協程_爬蟲

爬蟲簡單實例
from urllib import request
def f(url):
    print('GET: %s' % url)
    resp = request.urlopen(url)
    data = resp.read()
    f=open("url.html",'wb')
    f.write(data)
    f.close()
    print('%d bytes received from %s.' % (len(data), url))

f("https://www.cnblogs.com/alex3714/articles/5248247.html")

 

爬蟲實例
 from urllib import request
 import gevent,time
 from gevent import monkey
 monkey.patch_all() #把當前程序所有的IO操作給我單獨的做上標記 相當於打上sleep，識別IO阻塞
 
 def f(url):
     print('GET: %s' % url)
     resp = request.urlopen(url)
     data = resp.read()
     print('%d bytes received from %s.' % (len(data), url))
 
 urls=['https://www.python.org/',
       'https://www.qq.com/',
       'https://github.com/']
 
 #同步 串行 進程執行
 start_time=time.time()
 for url in urls:
     f(url)
 print("同步cost:",time.time()-start_time)
 
 #異步 協程
 async_time_start=time.time()
 
 #開啓3個協程，執行f,傳f的參數
 gevent.joinall([
         gevent.spawn(f, 'https://www.python.org/'),
         gevent.spawn(f, 'https://www.qq.com/'),
         gevent.spawn(f, 'https://github.com/'),
 ])
 
 print("異步cost:",time.time()-async_time_start)