Skip to main content

Appendix-03.ThreadPool

from multiprocessing.pool import ThreadPool
# from multiprocessing.util import _logger, log_to_stderr

# _logger = log_to_stderr()

from threading import current_thread
def work(x):
    # print(f"---{current_thread().name}")
    print(pool._pool)
    return x*x

pool = ThreadPool(processes=2)
results = pool.map(work, [1, 2, 3, 4, 5, 6, 7, 8])
print(results)




[<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>][<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>]
[<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>]
[<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>]
[<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>]
[<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>]
[<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>]
[<DummyProcess(Thread-96 (worker), started daemon 6383054848)>, <DummyProcess(Thread-97 (worker), started daemon 6399881216)>]

[1, 4, 9, 16, 25, 36, 49, 64]
0.005
import threading

isinstance(pool._pool[0], threading.Thread)

True

divmod(4, 4)

(1, 0)

import multiprocessing as mp

mp.__all__
globals()
mp.context
pool._pool
import os


os.cpu_count()
import itertools

tuple(itertools.islice([1, 2, 3, 4], 5))
pool._pool
import os
os.cpu_count()
pool._processes
pool._repopulate_pool()
pool._worker_handler
pool._inqueue
from multiprocessing.pool import ThreadPool

def work(x):
    return x*x

pool = ThreadPool(processes=4)
results = pool.map(work, [1, 2, 3, 4])
print(results)

[1, 4, 9, 16]
import threading

threading.current_thread()

<_MainThread(MainThread, started 8464163008)>
print(threading._active)




def get_all_active_threads():
    return threading._active


threads1 = set(list(get_all_active_threads()))

# print(threads1)

# threading.get_ident()
{8464163008: <_MainThread(MainThread, started 8464163008)>, 6111358976: <Thread(IOPub, started daemon 6111358976)>, 6128185344: <Heartbeat(Heartbeat, started daemon 6128185344)>, 6146158592: <Thread(Thread-1 (_watch_pipe_fd), started daemon 6146158592)>, 6162984960: <Thread(Thread-2 (_watch_pipe_fd), started daemon 6162984960)>, 6179811328: <ControlThread(Control, started daemon 6179811328)>, 6196637696: <ShellChannelThread(Shell channel, started daemon 6196637696)>, 6213464064: <HistorySavingThread(IPythonHistorySavingThread, started 6213464064)>, 6230863872: <DummyProcess(Thread-4 (worker), started daemon 6230863872)>, 6247690240: <DummyProcess(Thread-5 (worker), started daemon 6247690240)>, 6264516608: <DummyProcess(Thread-6 (worker), started daemon 6264516608)>, 6281342976: <DummyProcess(Thread-7 (worker), started daemon 6281342976)>, 6298169344: <Thread(Thread-8 (_handle_workers), started daemon 6298169344)>, 6314995712: <Thread(Thread-9 (_handle_tasks), started daemon 6314995712)>, 6331822080: <Thread(Thread-10 (_handle_results), started daemon 6331822080)>}
def work_hai(x):
    return x*x + 2

pool2 = ThreadPool(processes=3)
results = pool2.map(work_hai, [1, 2, 3, 4])
print(results)
[3, 6, 11, 18]

threads2 = set(list(get_all_active_threads()))
threads1 - threads2
set()
threads2 - threads1
{6348648448, 6365474816, 6382301184, 6399127552, 6415953920, 6432780288}
threads1
{6111358976,
 6128185344,
 6146158592,
 6162984960,
 6179811328,
 6196637696,
 6213464064,
 6230863872,
 6247690240,
 6264516608,
 6281342976,
 6298169344,
 6314995712,
 6331822080,
 8464163008}
pool2._pool
[<DummyProcess(Thread-11 (worker), started daemon 6425309184)>,
 <DummyProcess(Thread-12 (worker), started daemon 6442135552)>,
 <DummyProcess(Thread-13 (worker), started daemon 12901707776)>]
pool._pool
[<DummyProcess(Thread-4 (worker), started daemon 6307524608)>,
 <DummyProcess(Thread-5 (worker), started daemon 6324350976)>,
 <DummyProcess(Thread-6 (worker), started daemon 6341177344)>,
 <DummyProcess(Thread-7 (worker), started daemon 6358003712)>]