diff options
Diffstat (limited to 'examples')
| -rw-r--r-- | examples/cc_oop_demo.py | 59 | ||||
| -rw-r--r-- | examples/fips_oop_demo.py | 74 |
2 files changed, 21 insertions, 112 deletions
diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py deleted file mode 100644 index 4a67c6ff..00000000 --- a/examples/cc_oop_demo.py +++ /dev/null @@ -1,59 +0,0 @@ -import logging -from datetime import datetime -from pathlib import Path - -from sec_certs.config.configuration import config -from sec_certs.dataset import CCDataset - -logger = logging.getLogger(__name__) - - -def main(): - file_handler = logging.FileHandler(config.log_filepath) - stream_handler = logging.StreamHandler() - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - file_handler.setFormatter(formatter) - stream_handler.setFormatter(formatter) - logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler]) - start = datetime.now() - - # Create empty dataset - dset = CCDataset({}, Path("./debug_dataset"), "cc_full_dataset", "sample dataset description") - - # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web(to_download=True) - - # explicitly dump to json - dset.to_json(dset.json_path) - - # Retrieve protection profile IDs - dset.process_protection_profiles() - - # Load dataset from JSON - new_dset = CCDataset.from_json("./debug_dataset/cc_full_dataset.json") - assert dset == new_dset - - # Download pdfs and update json - dset.download_all_pdfs() - - # Convert pdfs to text and update json - dset.convert_all_pdfs() - - # Extract data from txt files and update json - dset._extract_data() - - # transform to pandas DataFrame - # df = dset.to_pandas() - - # Compute heuristics on the dataset - dset._compute_heuristics() - - # Compute related cves - # dset.compute_related_cves() - - end = datetime.now() - logger.info(f"The computation took {(end-start)} seconds.") - - -if __name__ == "__main__": - main() diff --git a/examples/fips_oop_demo.py b/examples/fips_oop_demo.py index 26e81d83..73d5c5c5 100644 --- a/examples/fips_oop_demo.py +++ b/examples/fips_oop_demo.py @@ -1,70 +1,38 @@ import logging from datetime import datetime -from pathlib import Path - -import click from sec_certs.config.configuration import config -from sec_certs.dataset import FIPSAlgorithmDataset, FIPSDataset +from sec_certs.dataset import FIPSDataset logger = logging.getLogger(__name__) -@click.command() -@click.option("--config-file", help="Path to config file") -@click.option("--json-file", help="Path to dataset json file") -@click.option("--no-download-algs", help="Redo scan of html files", is_flag=True) -@click.option("--redo-web-scan", help="Redo scan of PDF files", is_flag=True) -@click.option("--redo-keyword-scan", help="Don't download algs", is_flag=True) -@click.option( - "--higher-precision-results", - help="Redo table search for certificates with high error rate. Behaviour undefined if used on a newly instantiated dataset.", - is_flag=True, -) -def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_scan, higher_precision_results): - logging.basicConfig(level=logging.INFO) +def main(): + file_handler = logging.FileHandler(config.log_filepath) + stream_handler = logging.StreamHandler() + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + file_handler.setFormatter(formatter) + stream_handler.setFormatter(formatter) + logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler]) start = datetime.now() - # Load config - config.load(config_file if config_file else "../sec_certs/settings.yaml") - - # Create empty dataset - dset = FIPSDataset({}, Path("../fips_dataset"), "sample_dataset", "sample dataset description") - - # this is for creating test dataset, usually with small number of pdfs - # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing') - - # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web(redo=redo_web_scan) - - logger.info(f"Finished parsing. Have dataset with {len(dset)} certificates.") - logger.info(f"Dataset saved to {dset.root_dir}/fips_full_dataset.json") - - logger.info("Converting pdfs") + # Full processing of FIPS dataset, fresh run + dset = FIPSDataset() + dset.get_certs_from_web() dset.convert_all_pdfs() + dset.pdf_scan() + dset.extract_certs_from_tables() + dset.finalize_results() - logger.info("Extracting keywords now.") - dset.pdf_scan(redo=redo_keyword_scan) - - logger.info(f"Finished extracting certificates for {len(dset.certs)} items.") - - logger.info("Searching for tables in pdfs") - - not_decoded_files = dset.extract_certs_from_tables(higher_precision_results) - - logger.info(f"Done. Files not decoded: {not_decoded_files}") - logger.info("Parsing algorithms") - if not no_download_algs: - aset = FIPSAlgorithmDataset({}, Path(dset.root_dir / "web/algorithms"), "algorithms", "sample algs") - aset.get_certs_from_web() - logger.info(f"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.") - - dset.algorithms = aset + # TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/210 and refactor this + # if not no_download_algs: + # aset.get_certs_from_web() + # logger.info(f"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.") + # dset.algorithms = aset - logger.info("finalizing results.") - dset.finalize_results() + # TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/211 and uncomment + # dset.plot_graphs(show=False) - dset.plot_graphs(show=False) end = datetime.now() logger.info(f"The computation took {(end - start)} seconds.") |