爬蟲練習（一）-初體驗-淘寶商品名稱價格爬取
思路：
1、首先需要一個接口
2、希望爬取多頁，要了解翻頁的機制
第二頁
第三頁
由此可知，淘寶默認44爲一頁
爬蟲思路：
要定義四個函數：
1、用來抓取網頁的內容
import re
import requests as rs
import os
#1、抓取網頁
def GetContent(url):
try:
head = {
'accept-encoding': 'gzip, deflate, sdch, br',
'accept-language': 'zh-CN,zh;q=0.8',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4620.400 QQBrowser/9.7.13014.400',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'cache-control': 'max-age=0',
'authority': 's.taobao.com',
'cookie': 'miid=2068212171251276260; cna=YvxMEvl+JhUCAXPnMSUJhymi; hng=CN%7Czh-CN%7CCNY%7C156; thw=cn; _m_h5_tk=b42f55e88611d73a88e78a1520bc4ced_1582442005441; _m_h5_tk_enc=e80ad35c3150848e5ad0173d474b40fe; enc=yZL87SLXumyKkPbZsmenpov9aRW8WoTj3UlVbKvYq%2BALmLFJgP48275Zva8%2FoG%2BbkM44qSCyOZCnPvlV6DbsFg%3D%3D; v=0; _samesite_flag_=true; _tb_token_=565deb30175ee; sgcookie=D75g13IFkErNuFojbANjg; unb=2602104768; uc3=lg2=Vq8l%2BKCLz3%2F65A%3D%3D&vt3=F8dBxd3wqiM01J46ZF8%3D&id2=UU6hRiHl6tC%2Frg%3D%3D&nk2=3v4ljD7ra5bC6d5BT9K9pQ%3D%3D; csg=4aff3b2b; lgc=%5Cu5EFA%5Cu8BAE%5Cu600E%5Cu4E48%5Cu53D6%5Cu540D%5Cu5B57%5Cu554A; t=e7ad534a64c25f8816fa93fe6f8c4863; cookie17=UU6hRiHl6tC%2Frg%3D%3D; dnk=%5Cu5EFA%5Cu8BAE%5Cu600E%5Cu4E48%5Cu53D6%5Cu540D%5Cu5B57%5Cu554A; skt=8bf9b2dcdb64dcfe; cookie2=10124cf623f791d2e29696d9343f83bc; existShop=MTU4MjYwNjU2MQ%3D%3D; uc4=nk4=0%403LcI4%2BzBT9saZmTdSg9oeiSOvQ9%2BQx8jKv0E&id4=0%40U2xsBh9eUxOA1C2mX6rQvlbLzc79; tracknick=%5Cu5EFA%5Cu8BAE%5Cu600E%5Cu4E48%5Cu53D6%5Cu540D%5Cu5B57%5Cu554A; mt=ci=55_1; _cc_=VFC%2FuZ9ajQ%3D%3D; tg=0; _l_g_=Ug%3D%3D; sg=%E5%95%8A87; _nk_=%5Cu5EFA%5Cu8BAE%5Cu600E%5Cu4E48%5Cu53D6%5Cu540D%5Cu5B57%5Cu554A; cookie1=V3oagUsOJ6T3Qw8UrlL%2BWQ9cXELGg5thneeqVyyIews%3D; tfstk=ceJFB7mxevHEhxGwit6PPus08x-dZ2tHqRSftIXS9WK7yGfhMjf-wogjViIuw; alitrackid=www.taobao.com; lastalitrackid=www.taobao.com; uc1=cookie16=URm48syIJ1yk0MX2J7mAAEhTuw%3D%3D&cookie21=U%2BGCWk%2F7ow0zmglPa33heg%3D%3D&cookie15=URm48syIIVrSKA%3D%3D&existShop=false&pas=0&cookie14=UoTUOLUnH0Ynwg%3D%3D&tag=8&lng=zh_CN; JSESSIONID=E8108D4668AF12DF30F1AEACA136D5CB; isg=BKSkEyKipkOMENJfRiZFL-4fYKKWPcingBvtLL7FMG8yaUQz5k2YN9rLKcHxqgD_; l=dB_LyZwgQOaEdPZCBOCanurza77OSLAYYuPzaNbMi_5Bq6Ts7N_Oo8K29F96VjS1ix8B4zcBghp9-etkZklM5f--g3fr4LQjJFrs8wHZexf..',
'referer': 'https://www.taobao.com/',
}
r = rs.get(url,headers = head)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
print('爬取失敗')
2、用來存儲目標信息
def Fillinfo(html,list):
try:
plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)
scl = re.findall(r'\"view_sales\"\:\"\d.*?\"',html)
nam= re.findall(r'\"raw_title\"\:\".*?\"',html)
for i in range(len(plt)):
real_price = eval(plt[i].split(':')[1])
real_sale = eval(scl[i].split(':')[1])
real_name = eval(nam[i].split(':')[1])
list.append([real_name,real_price,real_sale])
except:
print('存儲失敗')
3、用來打印和存到文件夾內
def PrintGOD(list,file_name):
os.chdir('E:\\Text\\')
count=1
count1=1
with open(file_name,'w') as f:
tpl = '{:4}\t{:20}\t{:10}\t{:4}'
tpl1 = '{:4}\t{:16}\t{:10}\t{:4}\n'
f.writelines(tpl1.format('序號','商品名','價格','銷量'))
for each_goods in list:
f.writelines(tpl1.format(count,each_goods[0],each_goods[1],each_goods[2],chr(12288)))
count+=1
print(tpl.format('序號','商品名','價格','銷量',chr(12288)))
for each_good in list:
print(tpl.format(count1,each_good[0],each_good[1],each_good[2],chr(12288)))
count1+=1
4、用來運行上述三個函數
def main():
start_url = 'https://s.taobao.com/search?q='
goods = 'TimeX'
depth = 3
list = []
file_name = 'TIMEX.txt'
for i in range(depth):
url = start_url+goods+'&s='+str(i*44)
html =GetContent(url)
Fillinfo(html,list)
PrintGOD(list,file_name)
main()
問題：
1、在獲取內容的函數內，若不加上headers，實際上爬取的頁面是淘寶的登陸頁面。
解決辦法：
搜索頁面按F12>>點擊Network>>找到最上面的search（如下圖）>>右鍵Copy as cURL(bash)
打開https://curl.trillworks.com/>>把複製的內容粘貼在左框>>複製右框的headers內容(到}爲止)>>回答代碼>>在requests.get(url,headers = headers)即可
2、正則表達式不太熟練
注意：
1、注意要在冒號前面加上r，注意 . :等符號，要轉義的話要加上
2、正確書寫正則表達式
3、打印處count應該設爲兩個不同的
4、注意打印時要用中文空格：chr(12288)