爬蟲入門三——爬取貼吧

需求：輸入吧名，輸入起始頁碼，輸入結束頁碼，然後再當前文件夾中創建一個以吧名爲名字的文件夾，裏面是每一頁的html內容，文件名是吧名_page.html

import urllib.request
import urllib.parse
import os
url = 'http://tieba.baidu.com/f?ie=utf-8'
ba_name = input('請輸入吧名：')
start_page = int(input('請輸入要爬去的起始頁碼'))
end_page = int(input('請輸入要爬去的結束頁碼：'))
#創建文件夾
os.mkdir(ba_name)
if not os.path.exists(ba_name):
    os.mkdir(ba_name)
#一次爬去每一頁數據
for page in range(start_page,end_page  + 1):
    # page就是當前頁
    #拼接url的過程
    data ={
        'kw': ba_name,
        'pn':(page - 1)*50
    }
    data = urllib.parse.urlencode(data)
    #生成指定的url
    url_t = url + data
    #print(url_t)
    headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36'
    }
    request = urllib.request.Request(url=url_t,headers=headers)
    print('第%s頁開始下載...' % page)
    response = urllib.request.urlopen(request)
    #生成文件名
    filename = ba_name + '_' + str(page) + '.html'
    #拼接文件路徑
    filepath = ba_name + '/' + filename
    #寫內容
    with open(filepath,'wb') as  fp:
        fp.write(response.read())
    print('第%s頁結束下載...' % page)