diff options
| author | Stanislav Boboň | 2020-12-18 15:35:13 +0100 |
|---|---|---|
| committer | Stanislav Boboň | 2020-12-18 15:35:13 +0100 |
| commit | 5f69599cf9b0753b03e2130aac630bb26846ff3b (patch) | |
| tree | 615a5e070e384a0e94e12d80c55bd3cd55f706c4 | |
| parent | cfb6267d768fdda0477902f2fe7521cc482532fc (diff) | |
| download | sec-certs-5f69599cf9b0753b03e2130aac630bb26846ff3b.tar.gz sec-certs-5f69599cf9b0753b03e2130aac630bb26846ff3b.tar.zst sec-certs-5f69599cf9b0753b03e2130aac630bb26846ff3b.zip | |
slightly changed dump times
| -rw-r--r-- | fips_oop_demo.py | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/fips_oop_demo.py b/fips_oop_demo.py index b83abd29..08aea468 100644 --- a/fips_oop_demo.py +++ b/fips_oop_demo.py @@ -10,10 +10,10 @@ def main(): start = datetime.now() # Create empty dataset - # dset = FIPSDataset({}, Path('./fips_dataset'), 'sample_dataset', 'sample dataset description') + dset = FIPSDataset({}, Path('./fips_dataset'), 'sample_dataset', 'sample dataset description') # this is for creating test dataset, usually with small number of pdfs - dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing') + # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing') # Load metadata for certificates from CSV and HTML sources dset.get_certs_from_web() @@ -23,16 +23,17 @@ def main(): dset.to_json(dset.root_dir / 'fips_full_dataset.json') logging.info(f'Dataset saved to {dset.root_dir}/fips_full_dataset.json') - logging.info("Extracting keywords now.") - + logging.info("Converting pdfs") dset.convert_all_pdfs() + dset.to_json(dset.root_dir / 'fips_full_dataset.json') + logging.info("Extracting keywords now.") dset.extract_keywords() + logging.info(f'Finished extracting certificates for {len(dset.certs)} items.') + logging.info("Dumping dataset again...") dset.to_json(dset.root_dir / 'fips_full_dataset.json') - logging.info(f'Finished extracting certificates for {len(dset.keywords)} items.') - logging.info("Searching for tables in pdfs") not_decoded_files = dset.extract_certs_from_tables() |