網絡爬蟲

##網絡爬蟲

在理想的狀態下, 所有的ICP(internet Content Provider) 都應該爲自己的網絡提供API接口來共享它們允許其他程序獲取的數據, 在這種情況下爬蟲就不是必需品, 國內比較有名的電商平臺(如淘寶, 京東等), 社交平臺(如QQ/微博/微信等)這些網站都提供了自己的Open Api, 但是這類Open Api通常會對可以抓取的數據頻率進行限制. 對於大多數的公司而言, 計時的獲取行業相關數據就是企業生存的重要環節之一, 然而大部分企業在行業數據方面的匱乏是其與生俱來的短板, 合理的利用爬蟲來獲取數據並從中提取出有價值的信息是至關重要的. 當然爬蟲還有很多重要的應用領域, 以下列舉了其中一部分.

#設置代理操作器
proxy = urllib.request.ProxyHandler({‘http’:‘http://58.251.251.203:8118’})
#構建新的強求器，覆蓋默認opener
opener = urllib.request.build_opener(proxy,urllib.request.HTTPHandler)

urllib.request.install_opener(opener)

response = urllib.request.urlopen(‘http://www.baidu.com/s?wd=ip’)

html_content = response.read().decode(‘utf-8’)

print(html_content)

#免費的天氣接口https://www.sojson.com/blog/305.html
import urllib.request
import json

url=‘http://t.weather.sojson.com/api/weather/city/101180101’
resp = urllib.request.urlopen(url)

if resp.code == 200:
weather_json = resp.read().decode(‘utf-8’)
#print(type(weather_json),weather_json)
weather_data= json.loads(weather_json)
data = weather_data[‘data’]
print(’\n\n’,data)

todo_humidity = data['shidu']
today_pm25 = data['pm25']
today_temperature = data['wendu']
print(f'今日溫度：{today_temperature}，溼度:{todo_humidity}')

# weather_data['data']

import urllib.request
import urllib.parse

import json

http://api.map.baidu.com/place/v2/suggestion?query=天安門&region=北京&city_limit=true&output=json&ak=你的ak //GET請求

origin_args={‘query’:‘ATM機’,‘region’:‘鄭州’,‘output’:‘json’,‘ak’:‘C8jTc5FZ9H3DTGBE8qkpCOtiPbDXHMx6’}
#將中文參數base64編碼
b64_args = urllib.parse.urlencode(origin_args)
print(b64_args)

resp = urllib.request.urlopen(‘http://api.map。baidu.com/place/v2/search ‘+’?’+b64_args)

base_url = ‘http://api.map.baidu.com/place/v2/search’
url = base_url+’?’+b64_args
print(‘拼好的url’,url)
resp = urllib.request.urlopen(url)
content_json = resp.read().decode()
print(content_json)

#json轉對象
content_obj = json.loads(content_json)
print(content_obj)
results = content_obj[‘results’]
for row in results:
print(row[‘name’],row[‘address’])

#糗事百科
#分析：請求url https://www.qiushibaike.com/hot/page/1/ 要抓取div class='article’小內容 urllib或requests，

import urllib.request
from fake_useragent import UserAgent
import re, requests

encoding=‘UTF-8’

for i in range(20):

base_url = ‘https://www.qiushibaike.com/hot/page/’

url =base_url+‘1’+’/’

print(url)

ua = UserAgent()

headers = {

‘User-Agent’:ua.random

}

req = urllib.request.Request(url,headers = headers)

resp =urllib.request.urlopen(req)

html_content = resp.read().decode(‘utf-8’)

print(html_content)

res = requests.get(
url= ‘https://www.qiushibaike.com/hot/page’+‘1’+’/’,
headers={‘User-Agent’:UserAgent().random}
).text

pattern = re.compile(r’

.?(.?)’,re.S) #re.S匹配是包括
results = re.findall(pattern, res)
for row in results:
print(row)