初識爬蟲

初識爬蟲

一、HTTP與HTTPS

1.1 應用架構

1.2 HTTP協議

1.3 HTTPS協議

二、爬蟲概念

三、會話技術

四、練習：利用socket下載圖片

import socket
import re

url = "http://image.baidu.com/search/index?z=&tn=baiduimage&word=%E5%8A%A8%E6%BC%AB&pn=0&ie=utf-8&oe=utf-8&lm=-1&fr=&se=&sme=&width=1920&height=1080&hd=1&latest=0&copyright=0"
client = socket.socket()
# 建立連接
client.connect(("image.baidu.com", 80))
# 構造報文: 請求方式 路徑 協議版本回車符換行符(\r\n)請求頭部
request = "GET /search/index?z=&tn=baiduimage&word=%E5%8A%A8%E6%BC%AB&pn=0&ie=utf-8&oe=utf-8&lm=-1&fr=&se=&sme=&width=1920&height=1080&hd=1&latest=0&copyright=0 HTTP/1.0\r\nHost:image.baidu.com\r\n\r\n"
# 發送消息給百度
client.send(request.encode())
# 循環接收返回的數據
data = b""
res = client.recv(1024)
while res:
    data += res
    res = client.recv(1024)
    # print(data)
# 匹配每個圖片的url
img_url = re.findall(r'"hoverURL":"(.+?jpg)"', data.decode(), re.S)
print(img_url)
url_list = []
host_list = []
for i in img_url:
    request_url = re.findall(r'http.+(/it.+)', i)
    url_list.append(request_url[0])
    host_url = re.findall(r'.+//(img.+com)', i)
    host_list.append(host_url[0])
print("url_list:", url_list)
print("host_list:", host_list)

for i in range(len(url_list)):
    img_client = socket.socket()
    request_img = "GET {} HTTP/1.0\r\nHost:{}\r\nReferer: {}\r\n\r\n".format(url_list[i], host_list[i], url)
    host = host_list[i]
    # print("host:", host, type(host))
    img_client.connect((host, 80))
    img_client.send(request_img.encode())
    res_img = img_client.recv(1024)
    data_img = b""
    while res_img:
        data_img += res_img
        res_img = img_client.recv(1024)
        with open("動漫{}.jpg".format(i), "wb")as f:
            f.write(re.findall(b'\r\n\r\n(.+)', data_img, re.S)[0])
        # print("data_img:", data_img)
# print("{},{}".format(url_list[0], host_list[0]))
# print(data_img)