爬蟲爬取csdn某一分類下的所有文章標題

python+selenium+redis.
整個程序是通過selenium不斷點擊“刷新“獲得新的題目然後存到redis中。
沒有selenium可以通過改變ip和瀏覽器取得同樣的效果。

# -*- coding: utf-8 -*-
"""
Spyder Editor
This is a temporary script file.
author： ParkJiYeon
update:18.05.25
"""

from selenium import webdriver
import time
from lxml import etree
from redis import StrictRedis
def GetTitle(url):
    browser = webdriver.Chrome()
    browser.get(url)
    while True:
        for i in range(2,15):
            new_url = "//*[@id='feedlist_id']/li["+str(i)+"]/div/div[1]/h2/a"
            lis = browser.find_element_by_xpath(new_url)
            redis.rpush('text',lis.text)
        button = browser.find_element_by_xpath("//*[@id='nav']/div/div/ul/li[6]/a")
        button.click()
        time.sleep(5)
    browser.close()
def ShowTitle():
    len =redis.llen('text')
    for i in range(1,len):
        print(redis.lindex('text',i).decode("utf-8"))
if __name__=="__main__":
    url="https://blog.csdn.net/nav/cloud"
    redis = StrictRedis()
    GetTitle(url)
    ShowTitle()