python多進程寫同一個list/dict

python2下的寫法

import time
from tqdm import tqdm
import multiprocessing as mp


def picklable_op(_class, *args):
    """
    多進程之間要使用pickle來序列化並傳遞一些數據。
    由於py2下實例方法並不能像py3一樣直接被pickle。
    所以需要對多進程對象進行封裝，使之可以在py2下被pickle。
    """
    return _class.proc_func(*args)


class OP():
    def __init__(self):
        # 直接調用 Manager 提供的 list() 和 dict()
        self.manager = mp.Manager
        self.mp_lst = self.manager().list()
        self.mp_dict = self.manager().dict()
        self.length = 64

    def proc_func(self, i, j):
        self.mp_lst.append(i)
        self.mp_dict[i] = j
        time.sleep(0.1)

    def flow(self):
        pool = mp.Pool(16)
        for i in range(self.length):
            pool.apply_async(picklable_op, args=(self, i, i*2))
        pool.close()
        pool.join()


if __name__ == '__main__':

    start_time = time.time()
    op = OP()
    op.flow()
    print(op.mp_lst)
    print(op.mp_dict)
    print(time.time() - start_time)

python3下的寫法

import time
from tqdm import tqdm
import multiprocessing as mp


class OP():
    def __init__(self):
        # 直接調用 Manager 提供的 list() 和 dict()
        self.manager = mp.Manager
        self.mp_lst = self.manager().list()
        self.mp_dict = self.manager().dict()
        self.length = 64

    def proc_func(self, i, j):
        self.mp_lst.append(i)
        self.mp_dict[i] = j
        time.sleep(0.1)

    def flow(self):
        pool = mp.Pool(16)
        for i in range(self.length):
            pool.apply_async(self.proc_func, args=(i, i*2))
        pool.close()
        pool.join()


if __name__ == '__main__':

    start_time = time.time()
    op = OP()
    op.flow()
    print(op.mp_lst)
    print(op.mp_dict)
    print(time.time() - start_time)

---

參考文獻

[1] 今天遇到的Python多線程、多進程中的幾個坑