【icourse163】學習python爬蟲的代碼整理

學習python爬蟲的代碼整理

代碼思路等均參考課程：http://www.icourse163.org/learn/BIT-1001870001?tid=1002236011

1 對於一般的網頁的抓取

def getHTML(url):
    try:
        response = requests.get(url,timeout = 30)
        response.raise_for_status() #如果狀態不是200，則返回“產生異常”
        response.encoding = response.apparent_encoding
        return response.text
    except:
        return "產生異常"

if __name__=="__main__":
        url = "https://item.jd.com/5181380.html"
        print(getHTML(url))

2 亞馬遜的例子

由於亞馬遜有來源審查功能，因此限制User-Agent字段爲Python的訪問,所以採用以下方法爬取亞馬遜商品：

def getHTMLAmazon(url):
    try:
        kv = {'User-Agent':'Mozilla/5.0'}
        response = requests.get(url, headers = kv)
        response.raise_for_status()
        response.encoding = response.apparent_encoding
        return response.text
    except:
        return "爬取失敗"   


if __name__=="__main__":
        url = "https://www.amazon.cn/gp/product/B01M8L5Z3Y"
        print(getHTMLAmazon(url))

3 提供百度搜索的接口

def getBaiduso(keyword):
    try:
        kv ={"wd":keyword}
        response = requests.get("http://www.baidu.com/s", params = kv)
        print response.request.url
        response.raise_for_status()
        return len(response.text)
    except:
        return "爬取失敗"

keyword = "Python"
print getBaiduso(keyword)

4 爬取圖片並保存

def getPic(url,root):
    path = root +url.split('/')[-1]
    try:
        if not os.path.exists(root):
            os.mkdir(root)
        if not os.path.exists(path):
            response = requests.get(url)
            response.raise_for_status()
            with open(path,"wb") as f:
                f.write(response.content)
                f.close()
                print "文件已保存"
        else:
            print "文件已存在"
    except:
        print "爬取失敗"

url = "http://image.nationalgeographic.com.cn/2017/0526/20170526025441983.jpg"
root = "E://Python dir//"
getPic(url,root)

5 查詢IP地址歸屬地

def getIP(ipaddress):
    try:
        kv ={"ip":ipaddress}
        response = requests.get("http://m.ip138.com/ip.asp", params = kv)
        print response.request.url
        response.raise_for_status()
        return response.text[-500:]
    except:
        return "爬取失敗"

ipaddress = "202.204.80.112"
print getIP(ipaddress)