xpath抓取代理IP並檢測IP的有效性

#coding:utf-8
import urllib
import urllib2
import requests
import time
from bs4 import BeautifulSoup
from lxml import etree
import multiprocessing
test_url='http://www.baidu.com/'
testStr = "wahaha"
_headers={'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, sdch',
'Accept-Language':'zh-CN,zh;q=0.8',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Host':'www.xicidaili.com',
'If-None-Match':'W/"b077743016dc54409ebe6b86ba7a869b"',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
}
_cookies=None
ip_port=[]
file = open('proxy1.txt' , 'w')
#_cookies=requests.get('http://www.xicidaili.com/nn/1',headers=_headers).cookies

def process(page):
    # for page in xrange (1,10):
    response=requests.get('http://www.xicidaili.com/nn/'+'%s' % page,headers=_headers)
    text=response.text
    html=etree.HTML(text)
    result=html.xpath('(//tr[@class="odd"]|//tr[@class=""])/td[2]')
    result1 = html.xpath('(//tr[@class="odd"]|//tr[@class=""])/td[3]')
    result2 = html.xpath('(//tr[@class="odd"]|//tr[@class=""])/td[6]')
    for i ,j ,k in zip(result,result1,result2):
        ip=i.text
        port=j.text
        protocol=k.text
        # print '%s:%s' %(ip,port)
        if protocol== 'HTTP' or protocol=='HTTPS':
            cookies = urllib2.HTTPCookieProcessor()
            proxyHandler = urllib2.ProxyHandler({"http": r'http://%s:%s' % (ip, port)})
            opener = urllib2.build_opener(cookies,proxyHandler)
            opener.addheaders = [('User-Agent',
                                  'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36')]
            # urllib2.install_opener(opener)
            try:
                #reqs=urllib2.urlopen(test_url,timeout=5)
                reqs=opener.open(test_url,timeout=5)
                result=reqs.read()
                if len(result)>1:
                    print (ip ,port, protocol)
                    ip_port.append((ip, port, protocol))
                    file.write(protocol+":"+ip+":"+port+"\n")
                else :
                    continue
                time.sleep(1)
            except:
                continue
if __name__== '__main__':
    for i in xrange(1, 50):
        p=multiprocessing.Process(target=process,args=(i,))
        p.start()
    file.close()