diff options
| author | Stanislav Boboň | 2020-12-21 23:16:03 +0100 |
|---|---|---|
| committer | Stanislav Boboň | 2020-12-21 23:16:03 +0100 |
| commit | 7defb3bf2dfb87bfc2e6552aeb607f86a9ff0e83 (patch) | |
| tree | cf29602542efa113e1458afc603511a533adc133 | |
| parent | 9da6b5e58e0f011f2e07b902c92c6904df08a177 (diff) | |
| download | sec-certs-7defb3bf2dfb87bfc2e6552aeb607f86a9ff0e83.tar.gz sec-certs-7defb3bf2dfb87bfc2e6552aeb607f86a9ff0e83.tar.zst sec-certs-7defb3bf2dfb87bfc2e6552aeb607f86a9ff0e83.zip | |
default parameters
| -rw-r--r-- | fips_oop_demo.py | 5 | ||||
| -rw-r--r-- | sec_certs/dataset.py | 2 |
2 files changed, 3 insertions, 4 deletions
diff --git a/fips_oop_demo.py b/fips_oop_demo.py index 0262addf..45cd92b4 100644 --- a/fips_oop_demo.py +++ b/fips_oop_demo.py @@ -2,7 +2,6 @@ from sec_certs.dataset import FIPSDataset, FIPSAlgorithmDataset from pathlib import Path from datetime import datetime import logging -from sec_certs.helpers import download_parallel def main(): @@ -16,7 +15,7 @@ def main(): # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing') # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web(False) + dset.get_certs_from_web() logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') # Dump dataset into JSON @@ -28,7 +27,7 @@ def main(): dset.to_json(dset.root_dir / 'fips_full_dataset.json') logging.info("Extracting keywords now.") - dset.extract_keywords(True) + dset.extract_keywords() logging.info(f'Finished extracting certificates for {len(dset.certs)} items.') logging.info("Dumping dataset again...") diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 1c147e61..f784f434 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -518,7 +518,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): # TODO: make this work for single certs object instead of the whole dataset, making it parallelizable # TODO: make this not create a whole new json - that way continuous processing can be done - def extract_keywords(self, redo): + def extract_keywords(self, redo=False): self.fragments_dir.mkdir(parents=True, exist_ok=True) if self.new_files > 0 or not (self.root_dir / 'fips_full_keywords.json').exists(): |