「python爬蟲之路day15」:多線程爬取低俗段子（來看看你的fzl時代！）

今天學習了爬百思不得姐姐的段子，其實有點懵，也覺得自己技術之路的遙遙，所以記錄一下，然後準備打王者榮耀。
來看代碼：

import requests
from lxml import etree
import csv
import threading
from queue import Queue
class BSSpider(threading.Thread):
    headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36"}
    def __init__(self,page_queue,joke_queue,*args,**kwargs):
        super(BSSpider,self).__init__(*args,**kwargs)
        self.base_domain='http://www.budejie.com'
        self.page_queue=page_queue
        self.joke_queue=joke_queue
    def run(self):
        while True:
            if self.page_queue.empty():
                break
            url=self.page_queue.get()
            response=requests.get(url,headers=self.headers)
            text=response.text
            html=etree.HTML(text)
            descs=html.xpath('//div[@class="j-r-list-c-desc"]')
            for desc in descs:
                jokes=desc.xpath(".//text()")
                joke="\n".join(jokes).strip()#各項鍊接，返回字符串
                link=self.base_domain+desc.xpath(".//a/@href")[0]
                self.joke_queue.put((joke,link))
            print("*"*30+"第%s頁下載完成！"%url.split('/')[-1]+"*"*30)

class BSWriter(threading.Thread):
    def __init__(self,gLock,writer,joke_queue,*args,**kwargs):
        super(BSWriter, self).__init__(*args, **kwargs)
        self.writer = writer
        self.joke_queue = joke_queue
        self.lock=gLock
    def run(self):
        while True:
            try:
                joke_info = self.joke_queue.get(timeout=4)
                joke,link = joke_info
                self.lock.acquire()
                self.writer.writerow((joke,link))
                self.lock.release()
                print('保存一條')
            except:
                break

def main():
    page_queue=Queue(10)
    joke_queue=Queue(500)
    gLock=threading.Lock()
    fp=open('bsbs.csv','a',newline='',encoding='utf-8')
    writer=csv.writer(fp)
    writer.writerow(('content','link'))#寫入標籤
    for x in range(1,11):
        url='http://www.budejie.com/text/%d'%x
        page_queue.put(url)
    for x in range(5):
        t=BSSpider(page_queue,joke_queue)
        t.start()
    for x in range(5):
        t=BSWriter(gLock,writer,joke_queue)
        t.start()
if __name__ == '__main__':
    main()

來看我們保存的段子咯：

現在看看當年一直看的東西感覺索然無味，其實並不是它們變了，而是我們出現了其的替代品而且我們的要求越來越高了。