aboutsummaryrefslogtreecommitdiffhomepage
path: root/sec_certs/parallel_processing.py
diff options
context:
space:
mode:
authoradamjanovsky2021-12-15 09:13:50 +0100
committerGitHub2021-12-15 09:13:50 +0100
commit2369023f77f779a6eb935bed5bbb83c23520b537 (patch)
treeae6f5f3fafe8eef6d2cd0936353efd9aa0ee91a7 /sec_certs/parallel_processing.py
parent8f48d490683ce3bb9d9b96581021168d5d128051 (diff)
parent26cdcb9840d53250586933f77a64ec1ec1e4e204 (diff)
downloadsec-certs-2369023f77f779a6eb935bed5bbb83c23520b537.tar.gz
sec-certs-2369023f77f779a6eb935bed5bbb83c23520b537.tar.zst
sec-certs-2369023f77f779a6eb935bed5bbb83c23520b537.zip
Merge branch 'dev' into fix/better-multiprocessing
Diffstat (limited to 'sec_certs/parallel_processing.py')
-rw-r--r--sec_certs/parallel_processing.py5
1 files changed, 2 insertions, 3 deletions
diff --git a/sec_certs/parallel_processing.py b/sec_certs/parallel_processing.py
index be7b0d4b..0e2ac175 100644
--- a/sec_certs/parallel_processing.py
+++ b/sec_certs/parallel_processing.py
@@ -1,6 +1,5 @@
-from tqdm import tqdm
+from sec_certs.helpers import tqdm
from concurrent.futures import ProcessPoolExecutor as ProcessPool, ThreadPoolExecutor as ThreadPool
-# from multiprocessing.pool import Pool, ThreadPool
from typing import Callable, Iterable, Optional
import time
@@ -9,7 +8,7 @@ def process_parallel(func: Callable, items: Iterable, max_workers: int,
use_threading: bool = True, progress_bar: bool = True, unpack: bool = False,
progress_bar_desc: Optional[str] = None):
- pool = ThreadPool(max_workers) if use_threading else ProcessPool(max_workers)
+ pool: Union[Pool, ThreadPool] = ThreadPool(max_workers) if use_threading else Pool(max_workers)
results = [pool.submit(func, *i) for i in items] if unpack else [pool.submit(func, i) for i in items]
if progress_bar is True and items: