From 7defb3bf2dfb87bfc2e6552aeb607f86a9ff0e83 Mon Sep 17 00:00:00 2001 From: Stanislav BoboĊˆ Date: Mon, 21 Dec 2020 23:16:03 +0100 Subject: default parameters --- fips_oop_demo.py | 5 ++--- sec_certs/dataset.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fips_oop_demo.py b/fips_oop_demo.py index 0262addf..45cd92b4 100644 --- a/fips_oop_demo.py +++ b/fips_oop_demo.py @@ -2,7 +2,6 @@ from sec_certs.dataset import FIPSDataset, FIPSAlgorithmDataset from pathlib import Path from datetime import datetime import logging -from sec_certs.helpers import download_parallel def main(): @@ -16,7 +15,7 @@ def main(): # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing') # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web(False) + dset.get_certs_from_web() logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') # Dump dataset into JSON @@ -28,7 +27,7 @@ def main(): dset.to_json(dset.root_dir / 'fips_full_dataset.json') logging.info("Extracting keywords now.") - dset.extract_keywords(True) + dset.extract_keywords() logging.info(f'Finished extracting certificates for {len(dset.certs)} items.') logging.info("Dumping dataset again...") diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 1c147e61..f784f434 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -518,7 +518,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): # TODO: make this work for single certs object instead of the whole dataset, making it parallelizable # TODO: make this not create a whole new json - that way continuous processing can be done - def extract_keywords(self, redo): + def extract_keywords(self, redo=False): self.fragments_dir.mkdir(parents=True, exist_ok=True) if self.new_files > 0 or not (self.root_dir / 'fips_full_keywords.json').exists(): -- cgit v1.3.1