先獲取首頁的源代碼
def get_one_page(url):
try:
headers={
'User-Agent':'Mozilla/5.0(Macintosh;intel Mac OS 10_11_4)Applewebkit/537.36(KHTML,like Gecko)Chrome/52.0.2743.116 Safari/537.36'
}
re=requests.get(url,headers=headers)
if re.status_code==200:
return re.text
return None
except RequestException:
return None
def parse_one_page(html):
pattern=re.compile('<dd>.*?board-index.*?>(.*?)</i>.*?data-src="(.*?)".*?class="name".*?">(.*?)</a>.*?class="star">(.*?)</p>.*?releasetime.*?>(.*?)</p>',re.S)
items=re.findall(pattern,html)
for item in items :
yield {
'index':item[0],
'image':item[1],
'title':item[2],
'actor':item[3].strip()[3:],
'time':item[4].strip()[5:]
}
def save_one_page(content):
with open('re.txt','a',encoding='utf8')as f:
print(type(json.dumps(content)))
f.write(json.dumps(content,ensure_ascii=False)+'\n')
def main(offest):
url = ('https://maoyan.com/board/4?offset='+str(offest))
html = get_one_page(url)
for item in parse_one_page(html):
save_one_page(item)
if __name__ == '__main__':
for i in range(10):
main(offest=i*10)
結果