如何在python多線程執行任務，讀取文件，批量發請求

python多線程執行任務，讀取文件，批量發請求

# !/usr/bin/python
# -*- coding: utf-8 -*-
import requests
import threading
import time
import Queue
import json, urllib2

# 添加線程  創建5個線程名
threadList = ["Thread-1", "Thread-2", "Thread-3", "Thread-4", "Thread-5"]
# 設置隊列長度
workQueue = Queue.Queue(300)
# 線程池
threads = []
start = time.time()


class myThread(threading.Thread):
    def __init__(self, name, q):
        threading.Thread.__init__(self)
        self.name = name
        self.q = q

    def run(self):
        print("Starting " + self.name)
        while True:
            try:
                crawler(self.name, self.q)
            except:
                break
        print("Exiting " + self.name)


# 創建新線程
for tName in threadList:
    thread = myThread(tName, workQueue)
    thread.start()
    threads.append(thread)


def crawler(threadName, q):
    textmod = {}
    textmod = json.dumps(textmod)
    header_dict = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko',
                   "Content-Type": "application/json"}

    # 執行多線程
    # 從隊列裏取出數據
    url = q.get(timeout=2)
    try:
        req = urllib2.Request(url=url, data=textmod, headers=header_dict)
        res = urllib2.urlopen(req)
        res = res.read()
        print("index success:" + str(pid) + "      " + res)
    except Exception as e:
        print(q.qsize(), threadName, "Error: ", e)


# 讀取數據，放入隊列
count = 0
filename = 'id.txt'
f = open(filename, 'r').readlines()
for id in f:
    pid = id.replace("\"", "").replace("\n", "")
    count = count + 1
    # 生產index
    indexUrl = "https://www.qq.com"
    url = indexUrl + '/aaa?id=' + pid
    workQueue.put(url)

# 等待所有線程完成
for t in threads:
    t.join()

end = time.time()
print('多線程批量執行時間爲：', end - start)

代碼非常簡單：

1、配置下需要啓動多少個線程，

2、然後修改讀取數據，放入隊列那裏的文件名和代碼，改成自己要執行的代碼段。

3、修改從隊列取出數據執行代碼，多個線程自己來取數據。