From 2e41cf49829a7e83f038dbc590abf320c9d2553c Mon Sep 17 00:00:00 2001 From: mmstanone Date: Thu, 11 Nov 2021 10:20:07 +0100 Subject: rules, serialization and processing --- sec_certs/parallel_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'sec_certs/parallel_processing.py') diff --git a/sec_certs/parallel_processing.py b/sec_certs/parallel_processing.py index 20a82f32..3b91f652 100644 --- a/sec_certs/parallel_processing.py +++ b/sec_certs/parallel_processing.py @@ -1,12 +1,13 @@ from tqdm import tqdm from multiprocessing.pool import Pool, ThreadPool -from typing import Callable, Iterable, Optional +from typing import Callable, Iterable, Optional, Union import time def process_parallel(func: Callable, items: Iterable, max_workers: int, callback: Optional[Callable] = None, use_threading: bool = True, progress_bar: bool = True, unpack: bool = False, progress_bar_desc: Optional[str] = None): + pool: Union[Pool, ThreadPool] if use_threading is True: pool = ThreadPool(max_workers) else: -- cgit v1.3.1 From 872b12f60af62bed0d6b29e33492379b7ba39070 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Wed, 15 Dec 2021 07:49:21 +0100 Subject: minor refactoring of types --- sec_certs/dataset/common_criteria.py | 4 ++-- sec_certs/helpers.py | 13 +++++-------- sec_certs/parallel_processing.py | 6 +----- sec_certs/sample/cve.py | 4 ++-- sec_certs/serialization/json.py | 8 ++++---- 5 files changed, 14 insertions(+), 21 deletions(-) (limited to 'sec_certs/parallel_processing.py') diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py index f1366fbd..2d343d00 100644 --- a/sec_certs/dataset/common_criteria.py +++ b/sec_certs/dataset/common_criteria.py @@ -348,9 +348,9 @@ class CCDataset(Dataset, ComplexSerializableType): """ html_sources = list(self.HTML_PRODUCTS_URL.keys()) if get_active is False: - html_sources = list(filter(lambda x: 'active' not in x, html_sources)) + html_sources = [x for x in html_sources if 'active' not in x] if get_archived is False: - html_sources = list(filter(lambda x: 'archived' not in x, html_sources)) + html_sources = [x for x in html_sources if 'archived' not in x] new_certs = {} for file in html_sources: diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 36413562..10ff1774 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -727,14 +727,11 @@ def compute_heuristics_version(cert_name: str) -> List[str]: # identified_versions = list(set([max(x, key=len) for x in re.findall(VERSION_PATTERN, cert_name, re.IGNORECASE | re.VERBOSE)])) # return identified_versions if identified_versions else ['-'] - matched = [] - for x in matched_strings: - found = re.search(normalizer, x) - if found is None: - raise RuntimeError("Nothing was found in match string - this should not be happening.") - matched.append(found.group()) - - return matched if matched_strings else ['-'] + if not matched_strings: + return ['-'] + + matched = [re.search(normalizer, x) for x in matched_strings] + return [x.group() for x in matched if x is not None] def tokenize_dataset(dset: List[str], keywords: Set[str]) -> np.ndarray: return np.array([tokenize(x, keywords) for x in dset]) diff --git a/sec_certs/parallel_processing.py b/sec_certs/parallel_processing.py index c0a4ea63..b21c4001 100644 --- a/sec_certs/parallel_processing.py +++ b/sec_certs/parallel_processing.py @@ -7,12 +7,8 @@ import time def process_parallel(func: Callable, items: Iterable, max_workers: int, callback: Optional[Callable] = None, use_threading: bool = True, progress_bar: bool = True, unpack: bool = False, progress_bar_desc: Optional[str] = None): - pool: Union[Pool, ThreadPool] - if use_threading is True: - pool = ThreadPool(max_workers) - else: - pool = Pool(max_workers) + pool: Union[Pool, ThreadPool] = ThreadPool(max_workers) if use_threading else Pool(max_workers) results = [pool.apply_async(func, (*i,), callback=callback) for i in items] if unpack else [pool.apply_async(func, (i, ), callback=callback) for i in items] if progress_bar is True and items: diff --git a/sec_certs/sample/cve.py b/sec_certs/sample/cve.py index 94924483..f63a8869 100644 --- a/sec_certs/sample/cve.py +++ b/sec_certs/sample/cve.py @@ -90,8 +90,8 @@ class CVE(PandasSerializableType, ComplexSerializableType): for x in lst: if x['vulnerable']: cpe_uri = x['cpe23Uri'] - version_start: Optional[Tuple[str, Any]] - version_end: Optional[Tuple[str, Any]] + version_start: Optional[Tuple[str, str]] + version_end: Optional[Tuple[str, str]] if 'versionStartIncluding' in x and x['versionStartIncluding']: version_start = ('including', x['versionStartIncluding']) elif 'versionStartExcluding' in x and x['versionStartExcluding']: diff --git a/sec_certs/serialization/json.py b/sec_certs/serialization/json.py index d2182699..1cd83cbc 100644 --- a/sec_certs/serialization/json.py +++ b/sec_certs/serialization/json.py @@ -26,12 +26,12 @@ class ComplexSerializableType: raise TypeError(f'Dict: {dct} on {cls.__mro__}') from e def to_json(self, output_path: Optional[Union[str, Path]] = None): - if output_path is None: - if not hasattr(self, 'json_path'): + if output_path is None and not hasattr(self, 'json_path'): raise ValueError(f'The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument.') - output_path = self.json_path # type: ignore - if output_path is None: + elif output_path is None and self.json_path is None: raise ValueError(f'The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument.') + elif output_path is None: + output_path = self.json_path # type: ignore with Path(output_path).open('w') as handle: json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False) -- cgit v1.3.1