diff options
44 files changed, 3029 insertions, 2152 deletions
diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..fb532ca0 --- /dev/null +++ b/.flake8 @@ -0,0 +1,23 @@ +[flake8] +max-line-length = 120 +exclude = + .git, + __pycache__, + build, + dist, + venv, + certsvenv, + .eggs, + scratches, +max-complexity = 10 + +# Ignore complexity for CLI +per-file-ignores = + cc_cli.py: C901, + fips_cli.py: C901 + +ignore = + E501, # line length, should be handleded by black + W503, # line break before binary operator, depracated + E203, # whitespace before :, not PEP8 compliant + C901, # Temporary fix, remove once https://github.com/crocs-muni/sec-certs/issues/146 is solved. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..0a625f68 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,50 @@ +name: Lint (MyPy, Black, isort, Flake8) +on: + push: + workflow_dispatch: +jobs: + run-mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name : Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - name: Install python dependencies + run: pip install -r requirements.txt + - name: Install mypy and types + run: | + pip install mypy + python3 -m pip install types-PyYAML + python3 -m pip install types-python-dateutil + python3 -m pip install types-requests + - name: Run mypy + run: mypy . + black: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: psf/black@stable + isort: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + - uses: isort/isort-action@master + with: + requirementsFiles: "requirements.txt dev_requirements.txt" + flake8-lint: + runs-on: ubuntu-latest + name: Flake8 + steps: + - name: Check out source repository + uses: actions/checkout@v2 + - name: Set up Python environment + uses: actions/setup-python@v1 + with: + python-version: "3.8" + - name: flake8 Lint + uses: py-actions/flake8@v2
\ No newline at end of file diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml deleted file mode 100644 index 764fe266..00000000 --- a/.github/workflows/mypy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: MyPy code analysis -on: - push: - workflow_dispatch: -jobs: - run-mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name : Setup python - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - name: Install python dependencies - run: pip install -r requirements.txt - - name: Install mypy and types - run: | - pip install mypy - python3 -m pip install types-PyYAML - python3 -m pip install types-python-dateutil - python3 -m pip install types-requests - - name: Run mypy - run: mypy --ignore-missing-imports *.py sec_certs/ diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index b557067d..00000000 --- a/.mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -plugins = numpy.typing.mypy_plugin
\ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..63527213 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/psf/black + rev: 21.12b0 + hooks: + - id: black + language_version: python3.9 + args: ['--check'] + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + args: ['--check-only'] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.920' + hooks: + - id: mypy + additional_dependencies: + - 'numpy' + - 'types-PyYAML' + - 'types-python-dateutil' + - 'types-requests' + - repo: https://github.com/pycqa/flake8 + rev: '4.0.1' + hooks: + - id: flake8 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7184033e..fc38d6f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,4 +11,29 @@ You contribution is warmly welcomed. You can help by: ## Branches - `dev` is the default branch against which all pull requests are to be made. -- `master` is protected branch where stable releases reside. Each push or PR to master should be properly tagged by [semantic version](https://semver.org/) as it will invoke actions to publish docker image and PyPi package.
\ No newline at end of file +- `master` is protected branch where stable releases reside. Each push or PR to master should be properly tagged by [semantic version](https://semver.org/) as it will invoke actions to publish docker image and PyPi package. + +## Quality assurance + +All commits shall pass the lint pipeline of the following tools: + +- Mypy (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) +- Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) +- isort (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/dev/pyproject.toml) for settings) +- Flake8 (see [].flake8](https://github.com/crocs-muni/sec-certs/blob/dev/.flake8) for settings) + +These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/dev/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you. + +It should thus suffice to: + +```bash +pip3 install -r ./dev_requirements.txt && +pre-commit install && +pre-commit run --all-files +``` + +To ivoke the tools manually, you can, in the repository root, use: +- Mypy: `mypy .` +- Black: `black --check .` (without the flag to reformat) +- isort: `isort --check-only .` (without the flag to actually fix the issue) +- Flake8: `flake8 .`
\ No newline at end of file @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -from typing import Optional, List -import click -from pathlib import Path import logging import sys from datetime import datetime +from pathlib import Path +from typing import List, Optional + +import click from sec_certs.config.configuration import config from sec_certs.dataset.common_criteria import CCDataset @@ -13,22 +14,44 @@ logger = logging.getLogger(__name__) @click.command() -@click.argument('actions', required=True, nargs=-1, type=click.Choice(['all', 'build', 'download', 'convert', 'analyze', 'maintenances'], case_sensitive=False)) -@click.option('-o', '--output', type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True), - help='Path where the output of the experiment will be stored. May overwrite existing content.') -@click.option('-c', '--config', 'configpath', default=None, type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), - help='Path to your own config yaml file that will override the default one.') -@click.option('-i', '--input', 'inputpath', type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), - help='If set, the actions will be performed on a CC dataset loaded from JSON from the input path.') -@click.option('-s', '--silent', is_flag=True, help='If set, will not print to stdout') -def main(configpath: Optional[str], actions: List[str], inputpath: Optional[Path], output: Optional[Path], silent: bool): +@click.argument( + "actions", + required=True, + nargs=-1, + type=click.Choice(["all", "build", "download", "convert", "analyze", "maintenances"], case_sensitive=False), +) +@click.option( + "-o", + "--output", + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True), + help="Path where the output of the experiment will be stored. May overwrite existing content.", +) +@click.option( + "-c", + "--config", + "configpath", + default=None, + type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), + help="Path to your own config yaml file that will override the default one.", +) +@click.option( + "-i", + "--input", + "inputpath", + type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), + help="If set, the actions will be performed on a CC dataset loaded from JSON from the input path.", +) +@click.option("-s", "--silent", is_flag=True, help="If set, will not print to stdout") +def main( + configpath: Optional[str], actions: List[str], inputpath: Optional[Path], output: Optional[Path], silent: bool +): """ Specify actions, sequence of one or more strings from the following list: [all, build, download, convert, analyze] If 'all' is specified, all actions run against the dataset. Otherwise, only selected actions will run in the correct order. """ file_handler = logging.FileHandler(config.log_filepath) stream_handler = logging.StreamHandler(sys.stderr) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(formatter) stream_handler.setFormatter(formatter) handlers: List[logging.StreamHandler] = [file_handler] @@ -37,7 +60,9 @@ def main(configpath: Optional[str], actions: List[str], inputpath: Optional[Path output = Path(output) if not inputpath and not output: - print('Error: You did not specify path to load the dataset from, nor did you specify where dataset can be stored.') + print( + "Error: You did not specify path to load the dataset from, nor did you specify where dataset can be stored." + ) sys.exit(1) if not silent: @@ -50,58 +75,75 @@ def main(configpath: Optional[str], actions: List[str], inputpath: Optional[Path try: config.load(Path(configpath)) except FileNotFoundError: - print('Error: Bad path to configuration file') + print("Error: Bad path to configuration file") sys.exit(1) except ValueError as e: - print(f'Error: Bad format of configuration file: {e}') + print(f"Error: Bad format of configuration file: {e}") - actions_set = {'build', 'download', 'convert', 'analyze'} if 'all' in actions else set(actions) + actions_set = {"build", "download", "convert", "analyze"} if "all" in actions else set(actions) - if inputpath and 'build' not in actions_set: + if inputpath and "build" not in actions_set: dset: CCDataset = CCDataset.from_json(Path(inputpath)) if output: - print(f'Warning: you provided both input and output paths. The dataset from input path will get copied to output path.') + print( + "Warning: you provided both input and output paths. The dataset from input path will get copied to output path." + ) dset.root_dir = output - if inputpath and 'build' in actions_set: - print(f'Warning: you wanted to build a dataset but you provided one in JSON -- that will be ignored. New one will be constructed at: {output}') + if inputpath and "build" in actions_set: + print( + f"Warning: you wanted to build a dataset but you provided one in JSON -- that will be ignored. New one will be constructed at: {output}" + ) - if 'build' in actions_set: + if "build" in actions_set: if output is None: raise RuntimeError("Output path was not provided.") - dset = CCDataset(certs={}, root_dir=output, name=f'CommonCriteria_dataset', description=f'Full CommonCriteria dataset snapshot {datetime.now().date()}') + dset = CCDataset( + certs={}, + root_dir=output, + name="CommonCriteria_dataset", + description=f"Full CommonCriteria dataset snapshot {datetime.now().date()}", + ) dset.get_certs_from_web() - elif 'build' not in actions_set and not inputpath: - print('Error: If you do not provide input parameter, you must use \'build\' action to build dataset first.') + elif "build" not in actions_set and not inputpath: + print("Error: If you do not provide input parameter, you must use 'build' action to build dataset first.") sys.exit(1) - if 'download' in actions_set: + if "download" in actions_set: if not dset.state.meta_sources_parsed: - print('Error: You want to download all pdfs, but the data from commoncriteria.org was not parsed. You must use \'build\' action first.') + print( + "Error: You want to download all pdfs, but the data from commoncriteria.org was not parsed. You must use 'build' action first." + ) sys.exit(1) dset.download_all_pdfs() - if 'convert' in actions_set: + if "convert" in actions_set: if not dset.state.pdfs_downloaded: - print('Error: You want to convert pdfs -> txt, but the pdfs were not downloaded. You must use \'download\' action first.') + print( + "Error: You want to convert pdfs -> txt, but the pdfs were not downloaded. You must use 'download' action first." + ) sys.exit(1) dset.convert_all_pdfs() - if 'analyze' in actions_set: + if "analyze" in actions_set: if not dset.state.pdfs_converted: - print('Error: You want to process txt documents of certificates, but pdfs were not converted. You must use \'convert\' action first.') + print( + "Error: You want to process txt documents of certificates, but pdfs were not converted. You must use 'convert' action first." + ) sys.exit(1) dset.analyze_certificates() - if 'maintenances' in actions_set: + if "maintenances" in actions_set: if not dset.state.meta_sources_parsed: - print('Error: You want to process maintenance updates, but the data from commoncriteria.org was not parsed. You must use \'build\' action first.') + print( + "Error: You want to process maintenance updates, but the data from commoncriteria.org was not parsed. You must use 'build' action first." + ) sys.exit(1) dset.process_maintenance_updates() end = datetime.now() - logger.info(f'The computation took {(end-start)} seconds.') + logger.info(f"The computation took {(end-start)} seconds.") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/dev_requirements.txt b/dev_requirements.txt index 02bdf728..edb5f93a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,8 @@ mypy types-PyYAML types-python-dateutil -types-requests
\ No newline at end of file +types-requests +black +isort==5.10.1 +flake8==4.0.1 +pre-commit
\ No newline at end of file diff --git a/examples/cc_cpe_labeling.py b/examples/cc_cpe_labeling.py index 57ec93c3..89527bbe 100644 --- a/examples/cc_cpe_labeling.py +++ b/examples/cc_cpe_labeling.py @@ -1,10 +1,10 @@ -from datetime import datetime import logging +from datetime import datetime from pathlib import Path -from sec_certs.dataset.common_criteria import CCDataset from sec_certs.config.configuration import config -from sec_certs.model.evaluation import get_validation_dgsts, evaluate +from sec_certs.dataset.common_criteria import CCDataset +from sec_certs.model.evaluation import evaluate, get_validation_dgsts logger = logging.getLogger(__name__) @@ -12,14 +12,14 @@ logger = logging.getLogger(__name__) def main(): file_handler = logging.FileHandler(config.log_filepath) stream_handler = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(formatter) stream_handler.setFormatter(formatter) logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler]) start = datetime.now() # Fetch dataset metadata from CC Website, don't download nor parse PDFS - dset = CCDataset({}, Path('./my_debug_dataset'), 'cc_full_dataset', 'Full CC dataset') + dset = CCDataset({}, Path("./my_debug_dataset"), "cc_full_dataset", "Full CC dataset") dset.get_certs_from_web(to_download=True) # Automatically match CPEs and CVEs @@ -27,21 +27,23 @@ def main(): dset.compute_related_cves() # Load dataset of ground truth CPE labels - dset.load_label_studio_labels(Path(__file__).parent.parent / 'data/manual_cpe_labels/cc.json') + dset.load_label_studio_labels(Path(__file__).parent.parent / "data/manual_cpe_labels/cc.json") # Limit dataset only to validation part - validation_dgsts = get_validation_dgsts(Path(__file__).parent.parent / 'data/validation_test_split/cc/validation.json') + validation_dgsts = get_validation_dgsts( + Path(__file__).parent.parent / "data/validation_test_split/cc/validation.json" + ) validation_certs = [x for x in dset if x.dgst in validation_dgsts] # Evaluate CPE matching performance metrics (on validation set) and dump classification report into json y_valid = [(x.heuristics.verified_cpe_matches) for x in validation_certs] - evaluate(validation_certs, y_valid, './my_debug_dataset/classification_report.json') + evaluate(validation_certs, y_valid, "./my_debug_dataset/classification_report.json") - logger.info(f'{dset.json_path} should now contain fully labeled dataset.') + logger.info(f"{dset.json_path} should now contain fully labeled dataset.") end = datetime.now() - logger.info(f'The computation took {(end-start)} seconds.') + logger.info(f"The computation took {(end-start)} seconds.") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py index 7a87bc14..46f4d882 100644 --- a/examples/cc_oop_demo.py +++ b/examples/cc_oop_demo.py @@ -1,9 +1,9 @@ -from pathlib import Path -from datetime import datetime import logging +from datetime import datetime +from pathlib import Path -from sec_certs.dataset.common_criteria import CCDataset from sec_certs.config.configuration import config +from sec_certs.dataset.common_criteria import CCDataset logger = logging.getLogger(__name__) @@ -11,14 +11,14 @@ logger = logging.getLogger(__name__) def main(): file_handler = logging.FileHandler(config.log_filepath) stream_handler = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(formatter) stream_handler.setFormatter(formatter) logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler]) start = datetime.now() # Create empty dataset - dset = CCDataset({}, Path('./debug_dataset'), 'cc_full_dataset', 'sample dataset description') + dset = CCDataset({}, Path("./debug_dataset"), "cc_full_dataset", "sample dataset description") # Load metadata for certificates from CSV and HTML sources dset.get_certs_from_web(to_download=True) @@ -30,7 +30,7 @@ def main(): dset.process_protection_profiles() # Load dataset from JSON - new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json') + new_dset = CCDataset.from_json("./debug_dataset/cc_full_dataset.json") assert dset == new_dset # Download pdfs and update json @@ -43,7 +43,7 @@ def main(): dset._extract_data() # transform to pandas DataFrame - df = dset.to_pandas() + # df = dset.to_pandas() # Compute heuristics on the dataset dset._compute_heuristics() @@ -52,8 +52,8 @@ def main(): # dset.compute_related_cves() end = datetime.now() - logger.info(f'The computation took {(end-start)} seconds.') + logger.info(f"The computation took {(end-start)} seconds.") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/examples/fips_cpe_labeling.py b/examples/fips_cpe_labeling.py index ba373c8c..5cab7829 100644 --- a/examples/fips_cpe_labeling.py +++ b/examples/fips_cpe_labeling.py @@ -1,8 +1,8 @@ -from datetime import datetime import logging -from sec_certs.dataset.fips import FIPSDataset +from datetime import datetime from pathlib import Path -from sec_certs.model.evaluation import get_validation_dgsts, evaluate + +from sec_certs.dataset.fips import FIPSDataset logger = logging.getLogger(__name__) @@ -14,7 +14,7 @@ def main(): # dset: FIPSDataset = FIPSDataset({}, Path('./my_debug_dataset'), 'sample_dataset', 'sample dataset description') # dset.get_certs_from_web(no_download_algorithms=True) - dset = FIPSDataset({}, Path('../fips_dataset'), 'sample_dataset', 'sample dataset description') + dset = FIPSDataset({}, Path("../fips_dataset"), "sample_dataset", "sample dataset description") dset.get_certs_from_web() # # Label CPE and compute related vulnerabilities # dset.compute_cpe_heuristics() @@ -31,11 +31,11 @@ def main(): # y_valid = [x.heuristics.verified_cpe_matches for x in validation_certs] # evaluate(validation_certs, y_valid, './my_debug_dataset/classification_report.json') - logger.info(f'{dset.json_path} should now contain fully labeled dataset.') + logger.info(f"{dset.json_path} should now contain fully labeled dataset.") end = datetime.now() - logger.info(f'The computation took {(end - start)} seconds.') + logger.info(f"The computation took {(end - start)} seconds.") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/examples/fips_oop_demo.py b/examples/fips_oop_demo.py index 3c30c86e..a84b1ac9 100644 --- a/examples/fips_oop_demo.py +++ b/examples/fips_oop_demo.py @@ -1,33 +1,36 @@ -from pathlib import Path -from datetime import datetime import logging +from datetime import datetime +from pathlib import Path + import click + +from sec_certs.config.configuration import config from sec_certs.dataset.fips import FIPSDataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.config.configuration import config - logger = logging.getLogger(__name__) @click.command() -@click.option('--config-file', help='Path to config file') -@click.option('--json-file', help='Path to dataset json file') -@click.option('--no-download-algs', help='Redo scan of html files', is_flag=True) -@click.option('--redo-web-scan', help='Redo scan of PDF files', is_flag=True) -@click.option('--redo-keyword-scan', help='Don\'t download algs', is_flag=True) -@click.option('--higher-precision-results', - help='Redo table search for certificates with high error rate. Behaviour undefined if used on a newly instantiated dataset.', - is_flag=True) +@click.option("--config-file", help="Path to config file") +@click.option("--json-file", help="Path to dataset json file") +@click.option("--no-download-algs", help="Redo scan of html files", is_flag=True) +@click.option("--redo-web-scan", help="Redo scan of PDF files", is_flag=True) +@click.option("--redo-keyword-scan", help="Don't download algs", is_flag=True) +@click.option( + "--higher-precision-results", + help="Redo table search for certificates with high error rate. Behaviour undefined if used on a newly instantiated dataset.", + is_flag=True, +) def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_scan, higher_precision_results): logging.basicConfig(level=logging.INFO) start = datetime.now() # Load config - config.load(config_file if config_file else '../sec_certs/settings.yaml') + config.load(config_file if config_file else "../sec_certs/settings.yaml") # Create empty dataset - dset = FIPSDataset({}, Path('../fips_dataset'), 'sample_dataset', 'sample dataset description') + dset = FIPSDataset({}, Path("../fips_dataset"), "sample_dataset", "sample dataset description") # this is for creating test dataset, usually with small number of pdfs # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing') @@ -35,8 +38,8 @@ def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_s # Load metadata for certificates from CSV and HTML sources dset.get_certs_from_web(redo=redo_web_scan) - logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') - logger.info(f'Dataset saved to {dset.root_dir}/fips_full_dataset.json') + logger.info(f"Finished parsing. Have dataset with {len(dset)} certificates.") + logger.info(f"Dataset saved to {dset.root_dir}/fips_full_dataset.json") logger.info("Converting pdfs") dset.convert_all_pdfs() @@ -44,7 +47,7 @@ def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_s logger.info("Extracting keywords now.") dset.pdf_scan(redo=redo_keyword_scan) - logger.info(f'Finished extracting certificates for {len(dset.certs)} items.') + logger.info(f"Finished extracting certificates for {len(dset.certs)} items.") logger.info("Searching for tables in pdfs") @@ -53,9 +56,9 @@ def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_s logger.info(f"Done. Files not decoded: {not_decoded_files}") logger.info("Parsing algorithms") if not no_download_algs: - aset = FIPSAlgorithmDataset({}, Path(dset.root_dir / 'web/algorithms'), 'algorithms', 'sample algs') + aset = FIPSAlgorithmDataset({}, Path(dset.root_dir / "web/algorithms"), "algorithms", "sample algs") aset.get_certs_from_web() - logger.info(f'Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.') + logger.info(f"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.") dset.algorithms = aset @@ -64,8 +67,8 @@ def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_s dset.plot_graphs(show=False) end = datetime.now() - logger.info(f'The computation took {(end - start)} seconds.') + logger.info(f"The computation took {(end - start)} seconds.") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/fips_cli.py b/fips_cli.py index a0be7e75..149d2f81 100755 --- a/fips_cli.py +++ b/fips_cli.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 -from typing import Optional, List, Set -import click -from pathlib import Path import logging import sys -import os from datetime import datetime +from pathlib import Path +from typing import List, Optional, Set -from sec_certs.config.configuration import config, DEFAULT_CONFIG_PATH +import click + +from sec_certs.config.configuration import DEFAULT_CONFIG_PATH, config from sec_certs.dataset.fips import FIPSDataset logger = logging.getLogger(__name__) @@ -48,9 +48,9 @@ logger = logging.getLogger(__name__) "-n", "--name", "json_name", - default=str(datetime.now().strftime("%d-%n-%Y-%H:%M:%S")) + '.json', + default=str(datetime.now().strftime("%d-%n-%Y-%H:%M:%S")) + ".json", type=str, - help="Name of the json object to be created in the <<output>> directory. Defaults to <<timestamp>>.json." + help="Name of the json object to be created in the <<output>> directory. Defaults to <<timestamp>>.json.", ) @click.option("--no-download-algs", "no_download_algs", help="Don't fetch new algorithm implementations", is_flag=True) @click.option("--redo-web-scan", "redo_web_scan", help="Redo HTML webpage scan from scratch", is_flag=True) @@ -116,13 +116,13 @@ def main( stream_handler.setFormatter(formatter) handlers: List[logging.StreamHandler] = [file_handler] - script_dir = os.path.dirname(os.path.realpath(__file__)) - if output: output = Path(output) if not inputpath and not output: - logger.error("You did not specify path to load the dataset from, nor did you specify where dataset can be stored.") + logger.error( + "You did not specify path to load the dataset from, nor did you specify where dataset can be stored." + ) sys.exit(1) if not silent: @@ -154,16 +154,20 @@ def main( ) r_actions |= {"build"} if "new-run" in actions else {"update"} if "all" in actions else set() - + actions = r_actions if "build" in actions and "update" in actions: - logger.error("'build' and 'update' cannot be specified at once. Use 'build' to create dataset from scratch, 'update' to update existing dataset.") + logger.error( + "'build' and 'update' cannot be specified at once. Use 'build' to create dataset from scratch, 'update' to update existing dataset." + ) if "build" in actions: assert output if inputpath: - logger.warning("Both 'build' and 'inputpath' specified. 'build' creates new dataset, 'inputpath' will be ignored.") + logger.warning( + "Both 'build' and 'inputpath' specified. 'build' creates new dataset, 'inputpath' will be ignored." + ) dset: FIPSDataset = FIPSDataset( certs={}, root_dir=output, @@ -182,18 +186,20 @@ def main( assert inputpath dset = FIPSDataset.from_json(inputpath) - + assert dset.algorithms - logger.info(f'Have dataset with {len(dset)} certs and {len(dset.algorithms)} algorithms.') + logger.info(f"Have dataset with {len(dset)} certs and {len(dset.algorithms)} algorithms.") if output: - logger.warning("You provided both inputpath and outputpath, dataset will be copied to outputpath (without data)") + logger.warning( + "You provided both inputpath and outputpath, dataset will be copied to outputpath (without data)" + ) dset.root_dir = output dset.to_json(output) - if 'update' in actions: + if "update" in actions: dset.get_certs_from_web(no_download_algorithms=no_download_algs, update=True, redo_web_scan=redo_web_scan) - + if "convert" in actions or "update" in actions: dset.convert_all_pdfs() @@ -202,7 +208,9 @@ def main( if "table-search" in actions or "update" in actions: if not higher_precision_results: - logger.info("You are using table search without higher precision results. It is advised to use the switch in the next run.") + logger.info( + "You are using table search without higher precision results. It is advised to use the switch in the next run." + ) dset.extract_certs_from_tables(high_precision=higher_precision_results) if "analysis" in actions: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..031110f1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.black] +line-length = 120 +exclude = ''' +/( + \.git + | \.mypy_cache + | \.tox + | venv + | \.venv + | _build + | buck-out + | build + | dist +)/ +''' + +[tool.isort] +multi_line_output=3 +include_trailing_comma=true +force_grid_wrap=0 +use_parentheses=true +ensure_newline_before_comments=true +line_length=120 +skip=["certsvenv", "build"] + +[tool.mypy] +plugins = ["numpy.typing.mypy_plugin"] +ignore_missing_imports = true +exclude="build/"
\ No newline at end of file diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py index 3d0ab08e..8d0637b8 100644 --- a/sec_certs/cert_rules.py +++ b/sec_certs/cert_rules.py @@ -2,456 +2,427 @@ import copy import re from typing import Dict, List, Pattern -REGEXEC_SEP = r'[ ,;\]”)(]' +REGEXEC_SEP = r"[ ,;\]”)(]" rules_cert_id = [ - 'BSI-DSZ-CC-[0-9]+?-[0-9]+', # German BSI - 'BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+-[0-9]+', # German BSI - 'BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+', # German BSI + "BSI-DSZ-CC-[0-9]+?-[0-9]+", # German BSI + "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+-[0-9]+", # German BSI + "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+", # German BSI # 'CC-Zert-.+?', - 'ANSSI(?:-|-CC-)[0-9]+?/[0-9]+', # French + "ANSSI(?:-|-CC-)[0-9]+?/[0-9]+", # French # 'ANSSI-CC-CER-F-.+?', # French - 'DCSSI-[0-9]+?/[0-9]+?', # French - 'Certification Report [0-9]+?/[0-9]+?', # French - 'Rapport de certification [0-9]+?/[0-9]+?', # French - 'NSCIB-CC-[0-9][0-9][0-9][0-9].+?', # Netherlands - 'NSCIB-CC-[0-9][0-9][0-9][0-9][0-9]*-CR', # Netherlands - 'NSCIB-CC-[0-9][0-9]-[0-9]+?-CR[0-9]+?', # Netherlands - 'SERTIT-[0-9]+?', # Norway - 'CCEVS-VR-(?:|VID)[0-9]+?-[0-9]+?', # US NSA + "DCSSI-[0-9]+?/[0-9]+?", # French + "Certification Report [0-9]+?/[0-9]+?", # French + "Rapport de certification [0-9]+?/[0-9]+?", # French + "NSCIB-CC-[0-9][0-9][0-9][0-9].+?", # Netherlands + "NSCIB-CC-[0-9][0-9][0-9][0-9][0-9]*-CR", # Netherlands + "NSCIB-CC-[0-9][0-9]-[0-9]+?-CR[0-9]+?", # Netherlands + "SERTIT-[0-9]+?", # Norway + "CCEVS-VR-(?:|VID)[0-9]+?-[0-9]+?", # US NSA # '[0-9][0-9\-]+?-CR', # Canada - 'CRP[0-9][0-9][0-9][0-9]*?', # UK CESG - 'CERTIFICATION REPORT No. P[0-9]+?', # UK CESG - '20[0-9][0-9]-[0-9]+-INF-[0-9]+?', # Spain - 'KECS-CR-[0-9]+?-[0-9]+?', # Korea - 'KECS-ISIS-[0-9]+?-[0-9][0-9][0-9][0-9]', # Korea - 'CRP-C[0-9]+?-[0-9]+?', # Japan - 'ISCB-[0-9]+?-RPT-[0-9]+?', # Malaysia - 'OCSI/CERT/.+?', # Italia - '[0-9\\.]+?/TSE-CCCS-[0-9]+?', # Turkis CCCS - 'BTBD-.+?', # Turkis CCCS + "CRP[0-9][0-9][0-9][0-9]*?", # UK CESG + "CERTIFICATION REPORT No. P[0-9]+?", # UK CESG + "20[0-9][0-9]-[0-9]+-INF-[0-9]+?", # Spain + "KECS-CR-[0-9]+?-[0-9]+?", # Korea + "KECS-ISIS-[0-9]+?-[0-9][0-9][0-9][0-9]", # Korea + "CRP-C[0-9]+?-[0-9]+?", # Japan + "ISCB-[0-9]+?-RPT-[0-9]+?", # Malaysia + "OCSI/CERT/.+?", # Italia + "[0-9\\.]+?/TSE-CCCS-[0-9]+?", # Turkis CCCS + "BTBD-.+?", # Turkis CCCS ] rules_vendor = [ - 'NXP', - 'Infineon', - 'Samsung', - '(?:STMicroelectronics|STM)', - 'Feitian', - 'Gemalto', - 'Gemplus', - 'Axalto', - '(?:Oberthur|OBERTHUR)', - '(?:Idemia|IDEMIA)', - r'(?:G\&D|G\+D|Giesecke\+Devrient|Giesecke \& Devrient)', - 'Philips', - 'Sagem', + "NXP", + "Infineon", + "Samsung", + "(?:STMicroelectronics|STM)", + "Feitian", + "Gemalto", + "Gemplus", + "Axalto", + "(?:Oberthur|OBERTHUR)", + "(?:Idemia|IDEMIA)", + r"(?:G\&D|G\+D|Giesecke\+Devrient|Giesecke \& Devrient)", + "Philips", + "Sagem", ] -rules_eval_facilities = [ - 'Serma Technologies', - 'THALES - CEACI' -] +rules_eval_facilities = ["Serma Technologies", "THALES - CEACI"] rules_protection_profiles = [ - 'BSI-(?:CC[-_]|)PP[-_]*.+?', - 'PP-SSCD.+?', - 'PP_DBMS_.+?' - # 'Protection Profile', - #'CCMB-20.+?', - 'BSI-CCPP-.+?', - 'ANSSI-CC-PP.+?', - 'WBIS_V[0-9]\\.[0-9]', - 'EHCT_V.+?' + "BSI-(?:CC[-_]|)PP[-_]*.+?", + "PP-SSCD.+?", + "PP_DBMS_.+?" + # 'Protection Profile', + # 'CCMB-20.+?', + "BSI-CCPP-.+?", + "ANSSI-CC-PP.+?", + "WBIS_V[0-9]\\.[0-9]", + "EHCT_V.+?", ] rules_technical_reports = [ - 'BSI[ ]*TR-[0-9]+?(?:-[0-9]+?|)', - 'BSI [0-9]+?', # German BSI document containing list of issued certificates in some period + "BSI[ ]*TR-[0-9]+?(?:-[0-9]+?|)", + "BSI [0-9]+?", # German BSI document containing list of issued certificates in some period ] rules_device_id = [ - 'G87-.+?', - 'ATMEL AT.+?', + "G87-.+?", + "ATMEL AT.+?", ] -rules_os = [ - 'STARCOS(?: [0-9\\.]+?|)', - 'JCOP[ ]*[0-9]' -] +rules_os = ["STARCOS(?: [0-9\\.]+?|)", "JCOP[ ]*[0-9]"] rules_standard_id = [ - 'FIPS ?(?:PUB )?[0-9]+-[0-9]+?', - 'FIPS ?(?:PUB )?[0-9]+?', - 'NIST SP [0-9]+-[0-9]+?[a-zA-Z]?', - 'PKCS[ #]*[1-9]+', - 'TLS[ ]*v[0-9\\.]+', - 'TLS[ ]*v[0-9\\.]+', - 'BSI-AIS[ ]*[0-9]+?', - 'AIS[ ]*[0-9]+?', - 'RFC[ ]*[0-9]+?', - 'ISO/IEC[ ]*[0-9]+[-]*[0-9]*', - 'ISO/IEC[ ]*[0-9]+:[ 0-9]+', - 'ISO/IEC[ ]*[0-9]+', - 'ICAO(?:-SAC|)', - '[Xx]\\.509', - 'RFC [0-9]+', - '(?:SCP|scp)[ \']*[0-9][0-9]', - 'CC[I]*MB-20[0-9]+?-[0-9]+?-[0-9]+?', # Common Criteria methodology - 'CCIMB-9[0-9]-[0-9]+?' # Common Criteria methodology old + "FIPS ?(?:PUB )?[0-9]+-[0-9]+?", + "FIPS ?(?:PUB )?[0-9]+?", + "NIST SP [0-9]+-[0-9]+?[a-zA-Z]?", + "PKCS[ #]*[1-9]+", + "TLS[ ]*v[0-9\\.]+", + "TLS[ ]*v[0-9\\.]+", + "BSI-AIS[ ]*[0-9]+?", + "AIS[ ]*[0-9]+?", + "RFC[ ]*[0-9]+?", + "ISO/IEC[ ]*[0-9]+[-]*[0-9]*", + "ISO/IEC[ ]*[0-9]+:[ 0-9]+", + "ISO/IEC[ ]*[0-9]+", + "ICAO(?:-SAC|)", + "[Xx]\\.509", + "RFC [0-9]+", + "(?:SCP|scp)[ ']*[0-9][0-9]", + "CC[I]*MB-20[0-9]+?-[0-9]+?-[0-9]+?", # Common Criteria methodology + "CCIMB-9[0-9]-[0-9]+?", # Common Criteria methodology old ] rules_security_level = [ - 'EAL[ ]*[0-9+]+?', - 'EAL[ ]*[0-9] augmented+?', - 'ITSEC[ ]*E[1-9]*.+?', + "EAL[ ]*[0-9+]+?", + "EAL[ ]*[0-9] augmented+?", + "ITSEC[ ]*E[1-9]*.+?", ] rules_security_assurance_components = [ - r'ACE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'ACM(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'ACO(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'ADO(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'ADV(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'AGD(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'ALC(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'ATE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'AVA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'AMA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'APE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'ASE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)' + r"ACE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"ACM(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"ACO(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"ADO(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"ADV(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"AGD(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"ALC(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"ATE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"AVA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"AMA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"APE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"ASE(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", ] rules_security_functional_components = [ - r'FAU(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FCO(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FCS(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FDP(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FIA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FMT(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FPR(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FPT(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FRU(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FTA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)', - r'FTP(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)' + r"FAU(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FCO(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FCS(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FDP(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FIA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FMT(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FPR(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FPT(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FRU(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FTA(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", + r"FTP(?:_[A-Z]{3,4}){1,2}(?:\.[0-9]|\.[0-9]\.[0-9]|)", ] rules_cc_claims = [ - r'D\.[\._\-A-Z]+?', # user Data - r'O\.[\._\-A-Z]+?', # Objectives - r'T\.[\._\-A-Z]+?', # Threats - r'A\.[\._\-A-Z]+?', # Assumptions - r'R\.[\._\-A-Z]+?', # Requirements - r'OT\.[\._\-A-Z]+?', # security objectives - r'OP\.[\._\-A-Z]+?', # OPerations - r'OE\.[\._\-A-Z]+?', # Objectives for the Environment - r'SA\.[\._\-A-Z]+?', # Security Aspects - r'OSP\.[\._\-A-Z]+?', # Organisational Security Policy + r"D\.[\._\-A-Z]+?", # user Data + r"O\.[\._\-A-Z]+?", # Objectives + r"T\.[\._\-A-Z]+?", # Threats + r"A\.[\._\-A-Z]+?", # Assumptions + r"R\.[\._\-A-Z]+?", # Requirements + r"OT\.[\._\-A-Z]+?", # security objectives + r"OP\.[\._\-A-Z]+?", # OPerations + r"OE\.[\._\-A-Z]+?", # Objectives for the Environment + r"SA\.[\._\-A-Z]+?", # Security Aspects + r"OSP\.[\._\-A-Z]+?", # Organisational Security Policy ] - rules_javacard = [ - #'(?:Java Card|JavaCard)', - #'(?:Global Platform|GlobalPlatform)', - r'(?:Java Card|JavaCard) [2-3]\.[0-9](?:\.[0-9]|)', - r'JC[2-3]\.[0-9](?:\.[0-9]|)', - r'(?:Java Card|JavaCard) \(version [2-3]\.[0-9](?:\.[0-9]|)\)', - r'(?:Global Platform|GlobalPlatform) [2-3]\.[0-9]\.[0-9]', - r'(?:Global Platform|GlobalPlatform) \(version [2-3]\.[0-9]\.[0-9]\)', + # '(?:Java Card|JavaCard)', + # '(?:Global Platform|GlobalPlatform)', + r"(?:Java Card|JavaCard) [2-3]\.[0-9](?:\.[0-9]|)", + r"JC[2-3]\.[0-9](?:\.[0-9]|)", + r"(?:Java Card|JavaCard) \(version [2-3]\.[0-9](?:\.[0-9]|)\)", + r"(?:Global Platform|GlobalPlatform) [2-3]\.[0-9]\.[0-9]", + r"(?:Global Platform|GlobalPlatform) \(version [2-3]\.[0-9]\.[0-9]\)", ] rules_javacard_api_consts = [ # javacard API constants - r'ALG_(?:PSEUDO_RANDOM|SECURE_RANDOM|TRNG|ALG_PRESEEDED_DRBG|FAST|KEYGENERATION)', - r'ALG_DES_[A-Z_0-9]+', - r'ALG_RSA_[A-Z_0-9]+', - r'ALG_DSA_[A-Z_0-9]+', - r'ALG_ECDSA_[A-Z_0-9]+', - r'ALG_AES_[A-Z_0-9]+', - r'ALG_HMAC_[A-Z_0-9]+', - r'ALG_KOREAN_[A-Z_0-9]+', - r'ALG_EC_[A-Z_0-9]+?', # may have false positives like XCP_CPB_ALG_EC_BPOOLCRV - r'ALG_SHA_[A-Z_0-9]+', - r'ALG_SHA3_[A-Z_0-9]+', - r'ALG_MD[A-Z_0-9]+', - r'ALG_RIPEMD[A-Z_0-9]+', - r'ALG_ISO3309_[A-Z_0-9]+', - r'ALG_XDH', - r'ALG_SM2', - r'ALG_SM3', - r'ALG_NULL', - r'ALG_TRNG', - r'ALG_NULL', - r'SIG_CIPHER_[A-Z_0-9]+', - r'CIPHER_[A-Z_0-9]+', - r'PAD_[A-Z_0-9]+', - r'TYPE_[A-Z_0-9]+', - r'LENGTH_[A-Z_0-9]+', - r'OWNER_PIN[A-Z_0-9]*', + r"ALG_(?:PSEUDO_RANDOM|SECURE_RANDOM|TRNG|ALG_PRESEEDED_DRBG|FAST|KEYGENERATION)", + r"ALG_DES_[A-Z_0-9]+", + r"ALG_RSA_[A-Z_0-9]+", + r"ALG_DSA_[A-Z_0-9]+", + r"ALG_ECDSA_[A-Z_0-9]+", + r"ALG_AES_[A-Z_0-9]+", + r"ALG_HMAC_[A-Z_0-9]+", + r"ALG_KOREAN_[A-Z_0-9]+", + r"ALG_EC_[A-Z_0-9]+?", # may have false positives like XCP_CPB_ALG_EC_BPOOLCRV + r"ALG_SHA_[A-Z_0-9]+", + r"ALG_SHA3_[A-Z_0-9]+", + r"ALG_MD[A-Z_0-9]+", + r"ALG_RIPEMD[A-Z_0-9]+", + r"ALG_ISO3309_[A-Z_0-9]+", + r"ALG_XDH", + r"ALG_SM2", + r"ALG_SM3", + r"ALG_NULL", + r"ALG_TRNG", + r"ALG_NULL", + r"SIG_CIPHER_[A-Z_0-9]+", + r"CIPHER_[A-Z_0-9]+", + r"PAD_[A-Z_0-9]+", + r"TYPE_[A-Z_0-9]+", + r"LENGTH_[A-Z_0-9]+", + r"OWNER_PIN[A-Z_0-9]*", # named curves - r'BRAINPOOLP[A-Z_0-9]+(?:R|T)1', - r'ED25519', - r'ED448', - r'FRP256V1', - r'SECP[0-9]*R1', - r'SM2', - r'X25519', - r'X448', + r"BRAINPOOLP[A-Z_0-9]+(?:R|T)1", + r"ED25519", + r"ED448", + r"FRP256V1", + r"SECP[0-9]*R1", + r"SM2", + r"X25519", + r"X448", ] rules_javacard_packages = [ # javacard packages - r'java\.[a-z\.]+', - r'javacard\.[a-z\.]+', - r'javacardx\.[a-z\.]+', - r'org\.[0-9a-z\.]+', - r'uicc\.[a-z\.]+', - r'com\.[0-9a-z\.]+', - r'de\.bsi\.[a-z\.]+', + r"java\.[a-z\.]+", + r"javacard\.[a-z\.]+", + r"javacardx\.[a-z\.]+", + r"org\.[0-9a-z\.]+", + r"uicc\.[a-z\.]+", + r"com\.[0-9a-z\.]+", + r"de\.bsi\.[a-z\.]+", ] rules_crypto_algs = [ - 'RSA[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)', - 'RSASSAPKCS1-[Vv]1_5', - 'SHA[-]*(?:160|224|256|384|512)', - 'AES[-]*(?:128|192|256|)', - 'SHA-1', - 'MD5', - 'HMAC', - 'HMAC-SHA-(?:160|224|256|384|512)', - '(Diffie-Hellman|DH)', - 'ECDH', - 'ECDSA', - 'EdDSA', - '[3T]?DES', - 'ECC', - 'DTRNG', - 'TRNG', - 'RN[GD]', - 'RBG', - 'PACE' + "RSA[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)", + "RSASSAPKCS1-[Vv]1_5", + "SHA[-]*(?:160|224|256|384|512)", + "AES[-]*(?:128|192|256|)", + "SHA-1", + "MD5", + "HMAC", + "HMAC-SHA-(?:160|224|256|384|512)", + "(Diffie-Hellman|DH)", + "ECDH", + "ECDSA", + "EdDSA", + "[3T]?DES", + "ECC", + "DTRNG", + "TRNG", + "RN[GD]", + "RBG", + "PACE", ] rules_block_cipher_modes = [ - 'ECB', - 'CBC', - 'CTR', - 'CFB', - 'OFB', - 'GCM', + "ECB", + "CBC", + "CTR", + "CFB", + "OFB", + "GCM", ] rules_ecc_curves = [ - '(?:Curve |curve |)P-(192|224|256|384|521)', - '(?:brainpool|BRAINPOOL)P[0-9]{3}[rkt][12]', - '(?:secp|sect|SECP|SECT)[0-9]+?[rk][12]', - '(?:ansit|ansip|ANSIP|ANSIT)[0-9]+?[rk][12]', - '(?:anssi|ANSSI)[ ]*FRP[0-9]+?v1', - 'prime[0-9]{3}v[123]', - 'c2[pto]nb[0-9]{3}[vw][123]', - 'FRP256v1', - 'Curve(25519|1174|4417|22103|67254|383187|41417)', - 'Ed(25519|448)', - 'ssc-(160|192|224|256|288|320|384|512)', - 'Tweedle(dee|dum)', - 'JubJub', - 'BLS(12|24)-[0-9]{3}' + "(?:Curve |curve |)P-(192|224|256|384|521)", + "(?:brainpool|BRAINPOOL)P[0-9]{3}[rkt][12]", + "(?:secp|sect|SECP|SECT)[0-9]+?[rk][12]", + "(?:ansit|ansip|ANSIP|ANSIT)[0-9]+?[rk][12]", + "(?:anssi|ANSSI)[ ]*FRP[0-9]+?v1", + "prime[0-9]{3}v[123]", + "c2[pto]nb[0-9]{3}[vw][123]", + "FRP256v1", + "Curve(25519|1174|4417|22103|67254|383187|41417)", + "Ed(25519|448)", + "ssc-(160|192|224|256|288|320|384|512)", + "Tweedle(dee|dum)", + "JubJub", + "BLS(12|24)-[0-9]{3}", ] rules_cplc = [ - 'IC[ ]*Fabricator', - 'IC[ ]*Type', - 'IC[ ]*Version', + "IC[ ]*Fabricator", + "IC[ ]*Type", + "IC[ ]*Version", ] rules_crypto_engines = [ - 'TORNADO', - 'SmartMX', - 'SmartMX2', - 'NesCrypt', + "TORNADO", + "SmartMX", + "SmartMX2", + "NesCrypt", ] rules_crypto_libs = [ - '(?:NesLib|NESLIB) [v]*[0-9.]+', - 'AT1 Secure .{1,30}? Library [v]*[0-9.]+', - 'AT1 Secure RSA/ECC/SHA library', - 'Crypto Library [v]*[0-9.]+', - 'ATMEL Toolbox [0-9.]+', - 'v1\\.02\\.013', # Infineon's ROCA-vulnerable library - 'OpenSSL', - 'LibreSSL', - 'BoringSSL', - 'MatrixSSL', - 'Nettle', - 'GnuTLS', - 'libtomcrypt', - 'BearSSL', - 'Botan', - 'Crypto\\+\\+', - 'wolfSSL', - 'mbedTLS', - 's2n', - 'NSS', - 'libgcrypt', - 'BouncyCastle', - 'cryptlib', - 'NaCl', - 'libsodium', - 'libsecp256k1' + "(?:NesLib|NESLIB) [v]*[0-9.]+", + "AT1 Secure .{1,30}? Library [v]*[0-9.]+", + "AT1 Secure RSA/ECC/SHA library", + "Crypto Library [v]*[0-9.]+", + "ATMEL Toolbox [0-9.]+", + "v1\\.02\\.013", # Infineon's ROCA-vulnerable library + "OpenSSL", + "LibreSSL", + "BoringSSL", + "MatrixSSL", + "Nettle", + "GnuTLS", + "libtomcrypt", + "BearSSL", + "Botan", + "Crypto\\+\\+", + "wolfSSL", + "mbedTLS", + "s2n", + "NSS", + "libgcrypt", + "BouncyCastle", + "cryptlib", + "NaCl", + "libsodium", + "libsecp256k1", ] -rules_IC_data_groups = [ - r'EF\.DG[1-9][0-6]?', - r'EF\.COM', - r'EF\.CardAccess', - r'EF\.SOD', - r'EF\.ChipSecurity' -] +rules_IC_data_groups = [r"EF\.DG[1-9][0-6]?", r"EF\.COM", r"EF\.CardAccess", r"EF\.SOD", r"EF\.ChipSecurity"] rules_defenses = [ - '[Mm]alfunction', - 'Leak-Inherent', - '[Pp]hysical [Pp]robing', - '[pP]hysical [tT]ampering', - '[Ss]ide.channels?', - 'SPA', - 'DPA', - 'DFA', - '[Ff]+ault [iI]nduction', - '[Ff]+ault [iI]njection', - 'ROCA', - '[tT]iming [aA]ttacks?' + "[Mm]alfunction", + "Leak-Inherent", + "[Pp]hysical [Pp]robing", + "[pP]hysical [tT]ampering", + "[Ss]ide.channels?", + "SPA", + "DPA", + "DFA", + "[Ff]+ault [iI]nduction", + "[Ff]+ault [iI]njection", + "ROCA", + "[tT]iming [aA]ttacks?", ] rules_certification_process = [ - '[oO]ut of [sS]cope', - '[\\.\\(].{0,100}?.[oO]ut of [sS]cope..{0,100}?[\\.\\)]', - '.{0,100}[oO]ut of [sS]cope.{0,100}', - '.{0,100}confidential document.{0,100}', - '[sS]ecurity [fF]unction SF\\.[a-zA-Z0-9_]', + "[oO]ut of [sS]cope", + "[\\.\\(].{0,100}?.[oO]ut of [sS]cope..{0,100}?[\\.\\)]", + ".{0,100}[oO]ut of [sS]cope.{0,100}", + ".{0,100}confidential document.{0,100}", + "[sS]ecurity [fF]unction SF\\.[a-zA-Z0-9_]", ] rules_vulnerabilities = [ - 'CVE-[0-9]+?-[0-9]+?', - 'CWE-[0-9]+?', + "CVE-[0-9]+?-[0-9]+?", + "CWE-[0-9]+?", ] rules_other = [ - 'library', + "library", # 'http[s]*://.+?/' ] rules_fips_remove_algorithm_ids = [ - -# --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) --- -# + added (and #id) everywhere + # --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) --- + # + added (and #id) everywhere r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*and[\s#]*\d+)?", r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*and[\s#]*\d+)?", r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*and[\s#]*\d+)?", r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*and[\s#]*\d+)?", - -# --- same as above, without hw or fw --- + # --- same as above, without hw or fw --- r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", - -# --- SHS/A - (bits) (method) ((cert #) numbers) --- + # --- SHS/A - (bits) (method) ((cert #) numbers) --- r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", - -# --- RSA (bits) (method) ((cert #)) --- + # --- RSA (bits) (method) ((cert #)) --- r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", - -# --- RSA (SSA) (PKCS) (version) (#) --- + # --- RSA (SSA) (PKCS) (version) (#) --- r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{4})?", r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{3})?", r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{2})?", r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{1})?", - -# --- AES (bits) (method) ((cert #)) --- + # --- AES (bits) (method) ((cert #)) --- r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", - -# --- Diffie Helman (CVL) ((cert #)) --- + # --- Diffie Helman (CVL) ((cert #)) --- r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{4})", r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{3})", r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{2})", r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{1})", - -# --- DRBG (bits) (method) (cert #) --- + # --- DRBG (bits) (method) (cert #) --- r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", - -# --- DES (bits) (method) (cert #) + # --- DES (bits) (method) (cert #) r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*?and[\s#]*?(\d+))?", r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*?and[\s#]*?(\d+))?", r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*?and[\s#]*?(\d+))?", r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*?and[\s#]*?(\d+))?", - -# --- DSA (bits) (method) (cert #) + # --- DSA (bits) (method) (cert #) r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", - -# --- platforms (#)+ - this is used in modification history --- + # --- platforms (#)+ - this is used in modification history --- r"[Pp]latforms? #\d+(?:#\d+|,| |-|and)*[^\n]*", - -# --- CVL (#) --- + # --- CVL (#) --- r"CVL[\s#]*?(\d{4})", r"CVL[\s#]*?(\d{3})", r"CVL[\s#]*?(\d{2})", r"CVL[\s#]*?(\d{1})", - -# --- PAA (#) --- + # --- PAA (#) --- r"PAA[: #]*?\d{4}", r"PAA[: #]*?\d{3}", r"PAA[: #]*?\d{2}", r"PAA[: #]*?\d{1}", - -# --- (#) Type --- + # --- (#) Type --- r"(?:#|cert\.?|sample|Cert\.?|Certificate)[\s#]*?(\d+)?\s*?(?:AES|SHS|SHA|RSA|HMAC|Diffie-Hellman|DRBG|DES|CVL)", - -# --- PKCS (#) --- + # --- PKCS (#) --- r"PKCS[\s]?#?\d+", - r"PKSC[\s]?#?\d+", # typo, #625 - -# --- # C and # A (just in case) --- + r"PKSC[\s]?#?\d+", # typo, #625 + # --- # C and # A (just in case) --- r"#\s+?[Cc]\d+", - r"#\s+?[Aa]\d+" + r"#\s+?[Aa]\d+", ] rules_fips_to_remove = [ -# --- random words found --- + # --- random words found --- r"[Ss]lot #\d", # a card slot, #2069 - r"[Ss]eals? ?\(?#\d - #\d", # #1232 - r"\[#\d*\]", # some certs use this as references + r"[Ss]eals? ?\(?#\d - #\d", # #1232 + r"\[#\d*\]", # some certs use this as references r"CSP ?#\d", # #2795 - r"[Pp]ower [Ss]upply #\d", # #604 - r"TEL #\d and #\d", # #3337 - r"#\d+ - #\d+", # labels, seals... #1232 - r"#\d+‐#?\d+", # labels, seals... #3530 - r"#\d+ to #?\d+", # labels, seals... #3058 - r"see #\d+", # labels, seals... #3058 + r"[Pp]ower [Ss]upply #\d", # #604 + r"TEL #\d and #\d", # #3337 + r"#\d+ - #\d+", # labels, seals... #1232 + r"#\d+‐#?\d+", # labels, seals... #3530 + r"#\d+ to #?\d+", # labels, seals... #3058 + r"see #\d+", # labels, seals... #3058 r"#\d+, ?#\d+", r"#?\d+ and #?\d+", r"label \(#\d+\)", r"[Ll]abel #\d+", r"\(#\d\)", - r"IETF[25\s]*RFC[26\s]*#\d+", # #3425 + r"IETF[25\s]*RFC[26\s]*#\d+", # #3425 r"Document # 540-105000-A1", - r"Certificate #2287-1 from EMCE Engineering", # ??? - r"[sS]cenarios?\s?#\d+", # 3789 - r"#\d+\s?\(\S\)", # 2159 + r"Certificate #2287-1 from EMCE Engineering", # ??? + r"[sS]cenarios?\s?#\d+", # 3789 + r"#\d+\s?\(\S\)", # 2159 ] rules_fips_cert = [ @@ -462,13 +433,11 @@ rules_fips_cert = [ r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{4})(?!\d)", r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{3})(?!\d)", r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2})(?!\d)", - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1})(?!\d)" + r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1})(?!\d)", ] # rule still too "general" -rules_fips_security_level = [ - r"[lL]evel (\d)" -] +rules_fips_security_level = [r"[lL]evel (\d)"] rules_fips_htmls = [ r"module-name\">\s*(?P<fips_module_name>[^<]*)", @@ -486,7 +455,7 @@ rules_fips_htmls = [ r"Allowed Algorithms[\s\S]*?\">\s*(?P<fips_allowed_algorithms>[^<]*)", r"Software Versions[\s\S]*?\">\s*(?P<fips_software>[^<]*)", r"Product URL[\s\S]*?\">\s*<a href=\"(?P<fips_url>.*)\"", - r"Vendor<\/h4>[\s\S]*?href=\".*?\">(?P<fips_vendor>.*?)<\/a>" + r"Vendor<\/h4>[\s\S]*?href=\".*?\">(?P<fips_vendor>.*?)<\/a>", ] @@ -494,26 +463,26 @@ rules_fips_htmls = [ # Common rules # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ common_rules = {} -common_rules['rules_os'] = rules_os -common_rules['rules_standard_id'] = rules_standard_id -common_rules['rules_security_level'] = rules_security_level -common_rules['rules_security_assurance_components'] = rules_security_assurance_components -common_rules['rules_security_functional_components'] = rules_security_functional_components -common_rules['rules_cc_claims'] = rules_cc_claims -common_rules['rules_javacard'] = rules_javacard -common_rules['rules_javacard_api_consts'] = rules_javacard_api_consts -common_rules['rules_javacard_packages'] = rules_javacard_packages -common_rules['rules_crypto_algs'] = rules_crypto_algs -common_rules['rules_block_cipher_modes'] = rules_block_cipher_modes -common_rules['rules_ecc_curves'] = rules_ecc_curves -common_rules['rules_cplc'] = rules_cplc -common_rules['rules_crypto_engines'] = rules_crypto_engines -common_rules['rules_crypto_libs'] = rules_crypto_libs -common_rules['rules_IC_data_groups'] = rules_IC_data_groups -common_rules['rules_defenses'] = rules_defenses -common_rules['rules_certification_process'] = rules_certification_process -common_rules['rules_vulnerabilities'] = rules_vulnerabilities -common_rules['rules_other'] = rules_other +common_rules["rules_os"] = rules_os +common_rules["rules_standard_id"] = rules_standard_id +common_rules["rules_security_level"] = rules_security_level +common_rules["rules_security_assurance_components"] = rules_security_assurance_components +common_rules["rules_security_functional_components"] = rules_security_functional_components +common_rules["rules_cc_claims"] = rules_cc_claims +common_rules["rules_javacard"] = rules_javacard +common_rules["rules_javacard_api_consts"] = rules_javacard_api_consts +common_rules["rules_javacard_packages"] = rules_javacard_packages +common_rules["rules_crypto_algs"] = rules_crypto_algs +common_rules["rules_block_cipher_modes"] = rules_block_cipher_modes +common_rules["rules_ecc_curves"] = rules_ecc_curves +common_rules["rules_cplc"] = rules_cplc +common_rules["rules_crypto_engines"] = rules_crypto_engines +common_rules["rules_crypto_libs"] = rules_crypto_libs +common_rules["rules_IC_data_groups"] = rules_IC_data_groups +common_rules["rules_defenses"] = rules_defenses +common_rules["rules_certification_process"] = rules_certification_process +common_rules["rules_vulnerabilities"] = rules_vulnerabilities +common_rules["rules_other"] = rules_other # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -521,21 +490,21 @@ common_rules['rules_other'] = rules_other # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # rules_security_target_class rules = {} -rules['rules_vendor'] = rules_vendor -rules['rules_cert_id'] = rules_cert_id -rules['rules_protection_profiles'] = rules_protection_profiles -rules['rules_technical_reports'] = rules_technical_reports -rules['rules_device_id'] = rules_device_id +rules["rules_vendor"] = rules_vendor +rules["rules_cert_id"] = rules_cert_id +rules["rules_protection_profiles"] = rules_protection_profiles +rules["rules_technical_reports"] = rules_technical_reports +rules["rules_device_id"] = rules_device_id rules.update(common_rules) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # For FIPS # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ fips_rules_base: Dict[str, List[str]] = {} -fips_rules_base['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids -fips_rules_base['rules_to_remove'] = rules_fips_to_remove -fips_rules_base['rules_security_level'] = rules_fips_security_level -fips_rules_base['rules_cert_id'] = rules_fips_cert +fips_rules_base["rules_fips_algorithms"] = rules_fips_remove_algorithm_ids +fips_rules_base["rules_to_remove"] = rules_fips_to_remove +fips_rules_base["rules_security_level"] = rules_fips_security_level +fips_rules_base["rules_cert_id"] = rules_fips_cert fips_common_rules = copy.deepcopy(common_rules) # make separate copy not to process cc rules by fips's re.compile fips_rules: Dict[str, List[Pattern[str]]] = {} diff --git a/sec_certs/config/configuration.py b/sec_certs/config/configuration.py index 670d8cc9..d06f2e41 100644 --- a/sec_certs/config/configuration.py +++ b/sec_certs/config/configuration.py @@ -1,36 +1,36 @@ -import yaml -from typing import Union +import json from pathlib import Path +from typing import Union + import jsonschema -import json +import yaml class Configuration(object): def load(self, filepath: Union[str, Path]): - with Path(filepath).open('r') as file: + with Path(filepath).open("r") as file: state = yaml.load(file, Loader=yaml.FullLoader) script_dir = Path(__file__).parent - with (Path(script_dir) / 'settings-schema.json').open('r') as file: + with (Path(script_dir) / "settings-schema.json").open("r") as file: schema = json.loads(file.read()) try: jsonschema.validate(state, schema) except jsonschema.exceptions.ValidationError as e: - print(f'{e}\n\nIn file {filepath}') + print(f"{e}\n\nIn file {filepath}") for k, v in state.items(): setattr(self, k, v) - def __getattribute__(self, key): res = object.__getattribute__(self, key) - if isinstance(res, dict) and 'value' in res: - return res['value'] + if isinstance(res, dict) and "value" in res: + return res["value"] return object.__getattribute__(self, key) -DEFAULT_CONFIG_PATH = Path(__file__).parent / 'settings.yaml' +DEFAULT_CONFIG_PATH = Path(__file__).parent / "settings.yaml" config = Configuration() config.load(DEFAULT_CONFIG_PATH) diff --git a/sec_certs/constants.py b/sec_certs/constants.py index 86f06933..45cc60c9 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -2,8 +2,8 @@ from enum import Enum RESPONSE_OK = 200 # TODO: types don't match -RETURNCODE_OK = 'ok' -RETURNCODE_NOK = 'nok' +RETURNCODE_OK = "ok" +RETURNCODE_NOK = "nok" REQUEST_TIMEOUT = 10 MIN_CORRECT_CERT_SIZE = 5000 @@ -12,44 +12,46 @@ MIN_CC_HTML_SIZE = 5000000 MIN_CC_CSV_SIZE = 700000 MIN_CC_PP_DATASET_SIZE = 2500000 + class CertFramework(Enum): - CC = 'Common Criteria' - FIPS = 'FIPS' + CC = "Common Criteria" + FIPS = "FIPS" + -FIPS_BASE_URL = 'https://csrc.nist.gov' -FIPS_MODULE_URL = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/' +FIPS_BASE_URL = "https://csrc.nist.gov" +FIPS_MODULE_URL = "https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/" -TAG_MATCH_COUNTER = 'count' -TAG_MATCH_MATCHES = 'matches' +TAG_MATCH_COUNTER = "count" +TAG_MATCH_MATCHES = "matches" -TAG_CERT_HEADER_PROCESSED = 'cert_header_processed' +TAG_CERT_HEADER_PROCESSED = "cert_header_processed" -TAG_CERT_ID = 'cert_id' -TAG_CC_SECURITY_LEVEL = 'cc_security_level' -TAG_CC_VERSION = 'cc_version' -TAG_CERT_LAB = 'cert_lab' -TAG_CERT_ITEM = 'cert_item' -TAG_CERT_ITEM_VERSION = 'cert_item_version' -TAG_DEVELOPER = 'developer' -TAG_REFERENCED_PROTECTION_PROFILES = 'ref_protection_profiles' -TAG_HEADER_MATCH_RULES = 'match_rules' -TAG_PP_TITLE = 'pp_title' -TAG_PP_GENERAL_STATUS = 'pp_general_status' -TAG_PP_VERSION_NUMBER = 'pp_version_number' -TAG_PP_ID = 'pp_id' -TAG_PP_ID_REGISTRATOR = 'pp_id_registrator' -TAG_PP_DATE = 'pp_date' -TAG_PP_AUTHORS = 'pp_authors' -TAG_PP_REGISTRATOR = 'pp_registrator' -TAG_PP_REGISTRATOR_SIMPLIFIED = 'pp_registrator_simplified' -TAG_PP_SPONSOR = 'pp_sponsor' -TAG_PP_EDITOR = 'pp_editor' -TAG_PP_REVIEWER = 'pp_reviewer' -TAG_KEYWORDS = 'keywords' +TAG_CERT_ID = "cert_id" +TAG_CC_SECURITY_LEVEL = "cc_security_level" +TAG_CC_VERSION = "cc_version" +TAG_CERT_LAB = "cert_lab" +TAG_CERT_ITEM = "cert_item" +TAG_CERT_ITEM_VERSION = "cert_item_version" +TAG_DEVELOPER = "developer" +TAG_REFERENCED_PROTECTION_PROFILES = "ref_protection_profiles" +TAG_HEADER_MATCH_RULES = "match_rules" +TAG_PP_TITLE = "pp_title" +TAG_PP_GENERAL_STATUS = "pp_general_status" +TAG_PP_VERSION_NUMBER = "pp_version_number" +TAG_PP_ID = "pp_id" +TAG_PP_ID_REGISTRATOR = "pp_id_registrator" +TAG_PP_DATE = "pp_date" +TAG_PP_AUTHORS = "pp_authors" +TAG_PP_REGISTRATOR = "pp_registrator" +TAG_PP_REGISTRATOR_SIMPLIFIED = "pp_registrator_simplified" +TAG_PP_SPONSOR = "pp_sponsor" +TAG_PP_EDITOR = "pp_editor" +TAG_PP_REVIEWER = "pp_reviewer" +TAG_KEYWORDS = "keywords" FIPS_NOT_AVAILABLE_CERT_SIZE = 10000 -FIPS_ALG_URL = 'https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/validation-search?searchMode=implementation&page=' +FIPS_ALG_URL = "https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/validation-search?searchMode=implementation&page=" -FILE_ERRORS_STRATEGY = 'surrogateescape' +FILE_ERRORS_STRATEGY = "surrogateescape" STOP_ON_UNEXPECTED_NUMS = False APPEND_DETAILED_MATCH_MATCHES = False -LINE_SEPARATOR = ' '
\ No newline at end of file +LINE_SEPARATOR = " " diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py index b92b72f0..b0116221 100644 --- a/sec_certs/dataset/common_criteria.py +++ b/sec_certs/dataset/common_criteria.py @@ -1,28 +1,29 @@ import copy import itertools +import json import locale import shutil import tempfile from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Dict, Iterator, Optional, Set, Union, List, Tuple, Mapping, ClassVar -import json +from typing import Callable, ClassVar, Dict, Iterator, List, Mapping, Optional, Set, Tuple, Union import numpy as np import pandas as pd -from bs4 import Tag, BeautifulSoup +from bs4 import BeautifulSoup, Tag -from sec_certs import helpers as helpers, parallel_processing as cert_processing +from sec_certs import helpers as helpers +from sec_certs import parallel_processing as cert_processing +from sec_certs.config.configuration import config from sec_certs.dataset.dataset import Dataset, logger -from sec_certs.serialization.json import ComplexSerializableType, serialize, CustomJSONDecoder -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.certificate import Certificate from sec_certs.dataset.protection_profile import ProtectionProfileDataset -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate -from sec_certs.config.configuration import config from sec_certs.model.dependency_finder import DependencyFinder +from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate +from sec_certs.sample.certificate import Certificate +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.protection_profile import ProtectionProfile +from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, serialize class CCDataset(Dataset, ComplexSerializableType): @@ -36,11 +37,17 @@ class CCDataset(Dataset, ComplexSerializableType): def __bool__(self): return any(vars(self)) - certs: Dict[str, 'CommonCriteriaCert'] + certs: Dict[str, "CommonCriteriaCert"] # TODO: Figure out how to type this. The problem is that this breaks covariance of the types, which mypy doesn't allow. - def __init__(self, certs: Mapping[str, 'Certificate'], root_dir: Path, name: str = 'dataset name', - description: str = 'dataset_description', state: Optional[DatasetInternalState] = None): + def __init__( + self, + certs: Mapping[str, "Certificate"], + root_dir: Path, + name: str = "dataset name", + description: str = "dataset_description", + state: Optional[DatasetInternalState] = None, + ): super().__init__(certs, root_dir, name, description) if state is None: @@ -51,15 +58,15 @@ class CCDataset(Dataset, ComplexSerializableType): yield from self.certs.values() def to_dict(self): - return {**{'state': self.state}, **super().to_dict()} + return {**{"state": self.state}, **super().to_dict()} def to_pandas(self): df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=CommonCriteriaCert.pandas_columns) - df = df.set_index('dgst') + df = df.set_index("dgst") df.not_valid_before = pd.to_datetime(df.not_valid_before, infer_datetime_format=True) df.not_valid_after = pd.to_datetime(df.not_valid_after, infer_datetime_format=True) - df = df.astype({'category': 'category', 'status': 'category', 'scheme': 'category'}) + df = df.astype({"category": "category", "status": "category", "scheme": "category"}) df = df.fillna(value=np.nan) return df @@ -67,110 +74,107 @@ class CCDataset(Dataset, ComplexSerializableType): @classmethod def from_dict(cls, dct: Dict): dset = super().from_dict(dct) - dset.state = copy.deepcopy(dct['state']) + dset.state = copy.deepcopy(dct["state"]) return dset - @Dataset.root_dir.setter # type: ignore + @Dataset.root_dir.setter # type: ignore def root_dir(self, new_dir: Union[str, Path]): old_dset = copy.deepcopy(self) - Dataset.root_dir.fset(self, new_dir) # type: ignore + Dataset.root_dir.fset(self, new_dir) # type: ignore self.set_local_paths() - if self.state and old_dset.root_dir != Path('..'): - logger.info(f'Changing root dir of partially processed dataset. All contents will get copied to {new_dir}') + if self.state and old_dset.root_dir != Path(".."): + logger.info(f"Changing root dir of partially processed dataset. All contents will get copied to {new_dir}") self.copy_dataset_contents(old_dset) self.to_json() - def copy_dataset_contents(self, old_dset: 'CCDataset'): + def copy_dataset_contents(self, old_dset: "CCDataset"): if old_dset.state.meta_sources_parsed: try: shutil.copytree(old_dset.web_dir, self.web_dir) except FileNotFoundError as e: - logger.warning(f'Attempted to copy non-existing file: {e}') + logger.warning(f"Attempted to copy non-existing file: {e}") if old_dset.state.pdfs_downloaded: try: shutil.copytree(old_dset.certs_dir, self.certs_dir) except FileNotFoundError as e: - logger.warning(f'Attempted to copy non-existing file: {e}') + logger.warning(f"Attempted to copy non-existing file: {e}") if old_dset.state.certs_analyzed: try: shutil.copytree(old_dset.auxillary_datasets_dir, self.auxillary_datasets_dir) except FileNotFoundError as e: - logger.warning(f'Attempted to copy non-existing file: {e}') + logger.warning(f"Attempted to copy non-existing file: {e}") @property def certs_dir(self) -> Path: - return self.root_dir / 'certs' + return self.root_dir / "certs" @property def reports_dir(self) -> Path: - return self.certs_dir / 'reports' + return self.certs_dir / "reports" @property def reports_pdf_dir(self) -> Path: - return self.reports_dir / 'pdf' + return self.reports_dir / "pdf" @property def reports_txt_dir(self) -> Path: - return self.reports_dir / 'txt' + return self.reports_dir / "txt" @property def targets_dir(self) -> Path: - return self.certs_dir / 'targets' + return self.certs_dir / "targets" @property def targets_pdf_dir(self) -> Path: - return self.targets_dir / 'pdf' + return self.targets_dir / "pdf" @property def targets_txt_dir(self) -> Path: - return self.targets_dir / 'txt' + return self.targets_dir / "txt" @property def pp_dataset_path(self) -> Path: - return self.auxillary_datasets_dir / 'pp_dataset.json' + return self.auxillary_datasets_dir / "pp_dataset.json" - BASE_URL: ClassVar[str] = 'https://www.commoncriteriaportal.org' + BASE_URL: ClassVar[str] = "https://www.commoncriteriaportal.org" HTML_PRODUCTS_URL = { - 'cc_products_active.html': BASE_URL + '/products/', - 'cc_products_archived.html': BASE_URL + '/products/index.cfm?archived=1', + "cc_products_active.html": BASE_URL + "/products/", + "cc_products_archived.html": BASE_URL + "/products/index.cfm?archived=1", } - HTML_LABS_URL = {'cc_labs.html': BASE_URL + '/labs'} + HTML_LABS_URL = {"cc_labs.html": BASE_URL + "/labs"} CSV_PRODUCTS_URL = { - 'cc_products_active.csv': BASE_URL + '/products/certified_products.csv', - 'cc_products_archived.csv': BASE_URL + '/products/certified_products-archived.csv', + "cc_products_active.csv": BASE_URL + "/products/certified_products.csv", + "cc_products_archived.csv": BASE_URL + "/products/certified_products-archived.csv", } PP_URL = { - 'cc_pp_active.html': BASE_URL + '/pps/', - 'cc_pp_collaborative.html': BASE_URL + '/pps/collaborativePP.cfm?cpp=1', - 'cc_pp_archived.html': BASE_URL + '/pps/index.cfm?archived=1', - } - PP_CSV = { - 'cc_pp_active.csv': BASE_URL + '/pps/pps.csv', - 'cc_pp_archived.csv': BASE_URL + '/pps/pps-archived.csv' + "cc_pp_active.html": BASE_URL + "/pps/", + "cc_pp_collaborative.html": BASE_URL + "/pps/collaborativePP.cfm?cpp=1", + "cc_pp_archived.html": BASE_URL + "/pps/index.cfm?archived=1", } + PP_CSV = {"cc_pp_active.csv": BASE_URL + "/pps/pps.csv", "cc_pp_archived.csv": BASE_URL + "/pps/pps-archived.csv"} @property def active_html_tuples(self) -> List[Tuple[str, Path]]: - return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if 'active' in y] + return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if "active" in y] @property def archived_html_tuples(self) -> List[Tuple[str, Path]]: - return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if 'archived' in y] + return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if "archived" in y] @property def active_csv_tuples(self) -> List[Tuple[str, Path]]: - return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if 'active' in y] + return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if "active" in y] @property def archived_csv_tuples(self) -> List[Tuple[str, Path]]: - return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if 'archived' in y] + return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if "archived" in y] @classmethod def from_web_latest(cls): with tempfile.TemporaryDirectory() as tmp_dir: - dset_path = Path(tmp_dir) / 'cc_latest_dataset.json' + dset_path = Path(tmp_dir) / "cc_latest_dataset.json" helpers.download_file(config.cc_latest_snapshot, dset_path) return cls.from_json(dset_path) @@ -178,7 +182,7 @@ class CCDataset(Dataset, ComplexSerializableType): for cert in self: cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir) - def _merge_certs(self, certs: Dict[str, 'CommonCriteriaCert'], cert_source: Optional[str] = None): + def _merge_certs(self, certs: Dict[str, "CommonCriteriaCert"], cert_source: Optional[str] = None): """ Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates """ @@ -189,7 +193,7 @@ class CCDataset(Dataset, ComplexSerializableType): for crt in certs_to_merge: self[crt.dgst].merge(crt, cert_source) - logger.info(f'Added {len(new_certs)} new and merged further {len(certs_to_merge)} certificates to the dataset.') + logger.info(f"Added {len(new_certs)} new and merged further {len(certs_to_merge)} certificates to the dataset.") def download_csv_html_resources(self, get_active: bool = True, get_archived: bool = True): self.web_dir.mkdir(parents=True, exist_ok=True) @@ -206,13 +210,17 @@ class CCDataset(Dataset, ComplexSerializableType): html_urls, html_paths = [x[0] for x in html_items], [x[1] for x in html_items] csv_urls, csv_paths = [x[0] for x in csv_items], [x[1] for x in csv_items] - logger.info('Downloading required csv and html files.') + logger.info("Downloading required csv and html files.") self._download_parallel(html_urls, html_paths) self._download_parallel(csv_urls, csv_paths) + @serialize def process_protection_profiles(self, to_download: bool = True, keep_metadata: bool = True): - logger.info('Processing protection profiles.') - constructor = {True: ProtectionProfileDataset.from_web, False: ProtectionProfileDataset.from_json} + logger.info("Processing protection profiles.") + constructor: Dict[bool, Callable[..., ProtectionProfileDataset]] = { + True: ProtectionProfileDataset.from_web, + False: ProtectionProfileDataset.from_json, + } if to_download is True and not self.auxillary_datasets_dir.exists(): self.auxillary_datasets_dir.mkdir() pp_dataset = constructor[to_download](self.pp_dataset_path) @@ -226,24 +234,25 @@ class CCDataset(Dataset, ComplexSerializableType): self.pp_dataset_path.unlink() @serialize - def get_certs_from_web(self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, - get_archived: bool = True): + def get_certs_from_web( + self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, get_archived: bool = True + ): """ Parses all metadata about certificates """ if to_download is True: self.download_csv_html_resources(get_active, get_archived) - logger.info('Adding CSV certificates to CommonCriteria dataset.') + logger.info("Adding CSV certificates to CommonCriteria dataset.") csv_certs = self._get_all_certs_from_csv(get_active, get_archived) - self._merge_certs(csv_certs, cert_source='csv') + self._merge_certs(csv_certs, cert_source="csv") # TODO: Someway along the way, 3 certificates get lost. Investigate and fix. - logger.info('Adding HTML certificates to CommonCriteria dataset.') + logger.info("Adding HTML certificates to CommonCriteria dataset.") html_certs = self._get_all_certs_from_html(get_active, get_archived) - self._merge_certs(html_certs, cert_source='html') + self._merge_certs(html_certs, cert_source="html") - logger.info(f'The resulting dataset has {len(self)} certificates.') + logger.info(f"The resulting dataset has {len(self)} certificates.") if not keep_metadata: shutil.rmtree(self.web_dir) @@ -251,62 +260,75 @@ class CCDataset(Dataset, ComplexSerializableType): self.set_local_paths() self.state.meta_sources_parsed = True - def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> Dict[str, 'CommonCriteriaCert']: + def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> Dict[str, "CommonCriteriaCert"]: """ Creates dictionary of new certificates from csv sources. """ csv_sources = list(self.CSV_PRODUCTS_URL.keys()) - csv_sources = [x for x in csv_sources if 'active' not in x or get_active] - csv_sources = [x for x in csv_sources if 'archived' not in x or get_archived] + csv_sources = [x for x in csv_sources if "active" not in x or get_active] + csv_sources = [x for x in csv_sources if "archived" not in x or get_archived] new_certs = {} for file in csv_sources: partial_certs = self._parse_single_csv(self.web_dir / file) - logger.info( - f'Parsed {len(partial_certs)} certificates from: {file}') + logger.info(f"Parsed {len(partial_certs)} certificates from: {file}") new_certs.update(partial_certs) return new_certs @staticmethod - def _parse_single_csv(file: Path) -> Dict[str, 'CommonCriteriaCert']: + def _parse_single_csv(file: Path) -> Dict[str, "CommonCriteriaCert"]: """ Using pandas, this parses a single CSV file. """ + def map_ip_to_hostname(url: str) -> str: if not url: return url - tokens = url.split('/') - relative_path = '/' + '/'.join(tokens[3:]) + tokens = url.split("/") + relative_path = "/" + "/".join(tokens[3:]) return CCDataset.BASE_URL + relative_path def _get_primary_key_str(row: Tag): - prim_key = row['category'] + row['cert_name'] + row['report_link'] + prim_key = row["category"] + row["cert_name"] + row["report_link"] return prim_key - if 'active' in str(file): - cert_status = 'active' + if "active" in str(file): + cert_status = "active" else: - cert_status = 'archived' + cert_status = "archived" - csv_header = ['category', 'cert_name', 'manufacturer', 'scheme', 'security_level', 'protection_profiles', - 'not_valid_before', 'not_valid_after', 'report_link', 'st_link', 'maintenance_date', - 'maintenance_title', 'maintenance_report_link', 'maintenance_st_link'] + csv_header = [ + "category", + "cert_name", + "manufacturer", + "scheme", + "security_level", + "protection_profiles", + "not_valid_before", + "not_valid_after", + "report_link", + "st_link", + "maintenance_date", + "maintenance_title", + "maintenance_report_link", + "maintenance_st_link", + ] # TODO: Now skipping bad lines, smarter heuristics to be built for dumb files - df = pd.read_csv(file, engine='python', encoding='windows-1252', error_bad_lines=False) + df = pd.read_csv(file, engine="python", encoding="windows-1252", error_bad_lines=False) df = df.rename(columns={x: y for (x, y) in zip(list(df.columns), csv_header)}) - df['is_maintenance'] = ~df.maintenance_title.isnull() - df = df.fillna(value='') + df["is_maintenance"] = ~df.maintenance_title.isnull() + df = df.fillna(value="") - df[['not_valid_before', 'not_valid_after', 'maintenance_date']] = df[ - ['not_valid_before', 'not_valid_after', 'maintenance_date']].apply(pd.to_datetime) + df[["not_valid_before", "not_valid_after", "maintenance_date"]] = df[ + ["not_valid_before", "not_valid_after", "maintenance_date"] + ].apply(pd.to_datetime) - df['dgst'] = df.apply(lambda row: helpers.get_first_16_bytes_sha256( - _get_primary_key_str(row)), axis=1) + df["dgst"] = df.apply(lambda row: helpers.get_first_16_bytes_sha256(_get_primary_key_str(row)), axis=1) - df_base = df.loc[df.is_maintenance == False].copy() - df_main = df.loc[df.is_maintenance == True].copy() + df_base = df.loc[~df.is_maintenance].copy() + df_main = df.loc[df.is_maintenance].copy() df_base.report_link = df_base.report_link.map(map_ip_to_hostname) df_base.st_link = df_base.st_link.map(map_ip_to_hostname) @@ -315,76 +337,95 @@ class CCDataset(Dataset, ComplexSerializableType): df_main.maintenance_st_link = df_main.maintenance_st_link.map(map_ip_to_hostname) n_all = len(df_base) - n_deduplicated = len(df_base.drop_duplicates(subset=['dgst'])) + n_deduplicated = len(df_base.drop_duplicates(subset=["dgst"])) if (n_dup := n_all - n_deduplicated) > 0: - logger.warning( - f'The CSV {file} contains {n_dup} duplicates by the primary key.') + logger.warning(f"The CSV {file} contains {n_dup} duplicates by the primary key.") - df_base = df_base.drop_duplicates(subset=['dgst']) + df_base = df_base.drop_duplicates(subset=["dgst"]) df_main = df_main.drop_duplicates() - profiles = {x.dgst: set([ProtectionProfile(y) for y in - helpers.sanitize_protection_profiles(x.protection_profiles)]) for x in - df_base.itertuples()} + profiles = { + x.dgst: set( + [ProtectionProfile(pp_name=y) for y in helpers.sanitize_protection_profiles(x.protection_profiles)] + ) + for x in df_base.itertuples() + } updates: Dict[str, Set] = {x.dgst: set() for x in df_base.itertuples()} for x in df_main.itertuples(): - updates[x.dgst].add(CommonCriteriaCert.MaintenanceReport(x.maintenance_date.date(), x.maintenance_title, - x.maintenance_report_link, - x.maintenance_st_link)) + updates[x.dgst].add( + CommonCriteriaCert.MaintenanceReport( + x.maintenance_date.date(), x.maintenance_title, x.maintenance_report_link, x.maintenance_st_link + ) + ) certs = { - x.dgst: CommonCriteriaCert(cert_status, x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level, - x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, - None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None), None, None, - None) for - x in - df_base.itertuples()} + x.dgst: CommonCriteriaCert( + cert_status, + x.category, + x.cert_name, + x.manufacturer, + x.scheme, + x.security_level, + x.not_valid_before, + x.not_valid_after, + x.report_link, + x.st_link, + None, + None, + profiles.get(x.dgst, None), + updates.get(x.dgst, None), + None, + None, + None, + ) + for x in df_base.itertuples() + } return certs - def _get_all_certs_from_html(self, get_active: bool, get_archived: bool) -> Dict[str, 'CommonCriteriaCert']: + def _get_all_certs_from_html(self, get_active: bool, get_archived: bool) -> Dict[str, "CommonCriteriaCert"]: """ Prepares dictionary of certificates from all html files. """ html_sources = list(self.HTML_PRODUCTS_URL.keys()) if get_active is False: - html_sources = [x for x in html_sources if 'active' not in x] + html_sources = [x for x in html_sources if "active" not in x] if get_archived is False: - html_sources = [x for x in html_sources if 'archived' not in x] + html_sources = [x for x in html_sources if "archived" not in x] new_certs = {} for file in html_sources: partial_certs = self._parse_single_html(self.web_dir / file) - logger.info( - f'Parsed {len(partial_certs)} certificates from: {file}') + logger.info(f"Parsed {len(partial_certs)} certificates from: {file}") new_certs.update(partial_certs) return new_certs @staticmethod - def _parse_single_html(file: Path) -> Dict[str, 'CommonCriteriaCert']: + def _parse_single_html(file: Path) -> Dict[str, "CommonCriteriaCert"]: """ Prepares a dictionary of certificates from a single html file. """ def _get_timestamp_from_footer(footer): - locale.setlocale(locale.LC_ALL, 'en_US') + locale.setlocale(locale.LC_ALL, "en_US") footer_text = list(footer.stripped_strings)[0] - date_string = footer_text.split(',')[1:3] - time_string = footer_text.split(',')[3].split(' at ')[1] - formatted_datetime = date_string[0] + \ - date_string[1] + ' ' + time_string - return datetime.strptime(formatted_datetime, ' %B %d %Y %I:%M %p') + date_string = footer_text.split(",")[1:3] + time_string = footer_text.split(",")[3].split(" at ")[1] + formatted_datetime = date_string[0] + date_string[1] + " " + time_string + return datetime.strptime(formatted_datetime, " %B %d %Y %I:%M %p") - def _parse_table(soup: BeautifulSoup, cert_status: str, table_id: str, category_string: str) -> Dict[ - str, 'CommonCriteriaCert']: - tables = soup.find_all('table', id=table_id) + def _parse_table( + soup: BeautifulSoup, cert_status: str, table_id: str, category_string: str + ) -> Dict[str, "CommonCriteriaCert"]: + tables = soup.find_all("table", id=table_id) assert len(tables) <= 1 if not tables: return {} table = tables[0] - rows = list(table.find_all('tr')) - header, footer, body = rows[0], rows[1], rows[2:] + rows = list(table.find_all("tr")) + # header, footer = rows[0], rows[1] + body = rows[2:] # TODO: It's possible to obtain timestamp of the moment when the list was generated. It's identical for each table and should thus only be obtained once. Not necessarily in each table # timestamp = _get_timestamp_from_footer(footer) @@ -392,38 +433,39 @@ class CCDataset(Dataset, ComplexSerializableType): # TODO: Do we have use for number of expected certs? We get rid of duplicites, so no use for assert expected == actual # caption_str = str(table.findAll('caption')) # n_expected_certs = int(caption_str.split(category_string + ' – ')[1].split(' Certified Products')[0]) - table_certs = {x.dgst: x for x in [ - CommonCriteriaCert.from_html_row(row, cert_status, category_string) for row in body]} + table_certs = { + x.dgst: x for x in [CommonCriteriaCert.from_html_row(row, cert_status, category_string) for row in body] + } return table_certs - if 'active' in str(file): - cert_status = 'active' + if "active" in str(file): + cert_status = "active" else: - cert_status = 'archived' + cert_status = "archived" - cc_cat_abbreviations = ['AC', 'BP', 'DP', 'DB', 'DD', 'IC', 'KM', - 'MD', 'MF', 'NS', 'OS', 'OD', 'DG', 'TC'] - cc_table_ids = ['tbl' + x for x in cc_cat_abbreviations] - cc_categories = ['Access Control Devices and Systems', - 'Boundary Protection Devices and Systems', - 'Data Protection', - 'Databases', - 'Detection Devices and Systems', - 'ICs, Smart Cards and Smart Card-Related Devices and Systems', - 'Key Management Systems', - 'Mobility', - 'Multi-Function Devices', - 'Network and Network-Related Devices and Systems', - 'Operating Systems', - 'Other Devices and Systems', - 'Products for Digital Signatures', - 'Trusted Computing' - ] + cc_cat_abbreviations = ["AC", "BP", "DP", "DB", "DD", "IC", "KM", "MD", "MF", "NS", "OS", "OD", "DG", "TC"] + cc_table_ids = ["tbl" + x for x in cc_cat_abbreviations] + cc_categories = [ + "Access Control Devices and Systems", + "Boundary Protection Devices and Systems", + "Data Protection", + "Databases", + "Detection Devices and Systems", + "ICs, Smart Cards and Smart Card-Related Devices and Systems", + "Key Management Systems", + "Mobility", + "Multi-Function Devices", + "Network and Network-Related Devices and Systems", + "Operating Systems", + "Other Devices and Systems", + "Products for Digital Signatures", + "Trusted Computing", + ] cat_dict = {x: y for (x, y) in zip(cc_table_ids, cc_categories)} - with file.open('r') as handle: - soup = BeautifulSoup(handle, 'html5lib') + with file.open("r") as handle: + soup = BeautifulSoup(handle, "html5lib") certs = {} for key, val in cat_dict.items(): @@ -434,36 +476,40 @@ class CCDataset(Dataset, ComplexSerializableType): def _download_reports(self, fresh=True): self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh) and x.report_link] - cert_processing.process_parallel(CommonCriteriaCert.download_pdf_report, - certs_to_process, - config.n_threads, - progress_bar_desc='Downloading reports') + cert_processing.process_parallel( + CommonCriteriaCert.download_pdf_report, + certs_to_process, + config.n_threads, + progress_bar_desc="Downloading reports", + ) def _download_targets(self, fresh=True): self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh)] - cert_processing.process_parallel(CommonCriteriaCert.download_pdf_target, - certs_to_process, - config.n_threads, - progress_bar_desc='Downloading targets') + cert_processing.process_parallel( + CommonCriteriaCert.download_pdf_target, + certs_to_process, + config.n_threads, + progress_bar_desc="Downloading targets", + ) @serialize def download_all_pdfs(self, fresh: bool = True): if self.state.meta_sources_parsed is False: - logger.error('Attempting to download pdfs while not having csv/html meta-sources parsed. Returning.') + logger.error("Attempting to download pdfs while not having csv/html meta-sources parsed. Returning.") return - logger.info('Downloading CC sample reports') + logger.info("Downloading CC sample reports") self._download_reports(fresh) - logger.info('Downloading CC security targets') + logger.info("Downloading CC security targets") self._download_targets(fresh) if fresh is True: - logger.info('Attempting to re-download failed report links.') + logger.info("Attempting to re-download failed report links.") self._download_reports(False) - logger.info('Attempting to re-download failed security target links.') + logger.info("Attempting to re-download failed security target links.") self._download_targets(False) self.state.pdfs_downloaded = True @@ -471,139 +517,155 @@ class CCDataset(Dataset, ComplexSerializableType): def _convert_reports_to_txt(self, fresh: bool = True): self.reports_txt_dir.mkdir(parents=True, exist_ok=True) certs_to_process = [x for x in self if x.state.report_is_ok_to_convert(fresh)] - cert_processing.process_parallel(CommonCriteriaCert.convert_report_pdf, - certs_to_process, - config.n_threads, - progress_bar_desc='Converting reports to txt') + cert_processing.process_parallel( + CommonCriteriaCert.convert_report_pdf, + certs_to_process, + config.n_threads, + progress_bar_desc="Converting reports to txt", + ) def _convert_targets_to_txt(self, fresh: bool = True): self.targets_txt_dir.mkdir(parents=True, exist_ok=True) certs_to_process = [x for x in self if x.state.st_is_ok_to_convert(fresh)] - cert_processing.process_parallel(CommonCriteriaCert.convert_target_pdf, - certs_to_process, - config.n_threads, - progress_bar_desc='Converting targets to txt') + cert_processing.process_parallel( + CommonCriteriaCert.convert_target_pdf, + certs_to_process, + config.n_threads, + progress_bar_desc="Converting targets to txt", + ) @serialize def convert_all_pdfs(self, fresh: bool = True): if self.state.pdfs_downloaded is False: - logger.info('Attempting to convert pdf while not having them downloaded. Returning.') + logger.info("Attempting to convert pdf while not having them downloaded. Returning.") return - logger.info('Converting CC sample reports to .txt') + logger.info("Converting CC sample reports to .txt") self._convert_reports_to_txt(fresh) - logger.info('Converting CC security targets to .txt') + logger.info("Converting CC security targets to .txt") self._convert_targets_to_txt(fresh) if fresh is True: - logger.info('Attempting to re-convert failed report pdfs') + logger.info("Attempting to re-convert failed report pdfs") self._convert_reports_to_txt(False) - logger.info('Attempting to re-convert failed target pdfs') + logger.info("Attempting to re-convert failed target pdfs") self._convert_targets_to_txt(False) self.state.pdfs_converted = True def update_with_certs(self, certs: List[CommonCriteriaCert]): if any([x not in self for x in certs]): - logger.warning('Updating dataset with certificates outside of the dataset!') + logger.warning("Updating dataset with certificates outside of the dataset!") self.certs.update({x.dgst: x for x in certs}) def _extract_report_metadata(self, fresh: bool = True): certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze(fresh)] - processed_certs = cert_processing.process_parallel(CommonCriteriaCert.extract_report_pdf_metadata, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc='Extracting report metadata') + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_report_pdf_metadata, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting report metadata", + ) self.update_with_certs(processed_certs) def _extract_targets_metadata(self, fresh: bool = True): certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze(fresh)] - processed_certs = cert_processing.process_parallel(CommonCriteriaCert.extract_st_pdf_metadata, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc='Extracting target metadata') + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_st_pdf_metadata, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting target metadata", + ) self.update_with_certs(processed_certs) def extract_pdf_metadata(self, fresh: bool = True): - logger.info('Extracting pdf metadata from CC dataset') + logger.info("Extracting pdf metadata from CC dataset") self._extract_report_metadata(fresh) self._extract_targets_metadata(fresh) def _extract_report_frontpage(self, fresh: bool = True): certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze(fresh)] - processed_certs = cert_processing.process_parallel(CommonCriteriaCert.extract_report_pdf_frontpage, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc='Extracting report frontpages') + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_report_pdf_frontpage, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting report frontpages", + ) self.update_with_certs(processed_certs) def _extract_targets_frontpage(self, fresh: bool = True): certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze(fresh)] - processed_certs = cert_processing.process_parallel(CommonCriteriaCert.extract_st_pdf_frontpage, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc='Extracting target frontpages') + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_st_pdf_frontpage, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting target frontpages", + ) self.update_with_certs(processed_certs) def extract_pdf_frontpage(self, fresh: bool = True): - logger.info('Extracting pdf frontpages from CC dataset.') + logger.info("Extracting pdf frontpages from CC dataset.") self._extract_report_frontpage(fresh) self._extract_targets_frontpage(fresh) def _extract_report_keywords(self, fresh: bool = True): certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze(fresh)] - processed_certs = cert_processing.process_parallel(CommonCriteriaCert.extract_report_pdf_keywords, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc='Extracting report keywords') + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_report_pdf_keywords, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting report keywords", + ) self.update_with_certs(processed_certs) def _extract_targets_keywords(self, fresh: bool = True): certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze(fresh)] - processed_certs = cert_processing.process_parallel(CommonCriteriaCert.extract_st_pdf_keywords, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc='Extracting target keywords') + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_st_pdf_keywords, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting target keywords", + ) self.update_with_certs(processed_certs) def extract_pdf_keywords(self, fresh: bool = True): - logger.info('Extracting pdf keywords from CC dataset.') + logger.info("Extracting pdf keywords from CC dataset.") self._extract_report_keywords(fresh) self._extract_targets_keywords(fresh) def _extract_data(self, fresh: bool = True): - logger.info('Extracting various stuff from converted txt filed from CC dataset.') + logger.info("Extracting various stuff from converted txt filed from CC dataset.") self.extract_pdf_metadata(fresh) self.extract_pdf_frontpage(fresh) self.extract_pdf_keywords(fresh) if fresh is True: - logger.info('Attempting to re-extract failed data from report txts') + logger.info("Attempting to re-extract failed data from report txts") self._extract_report_metadata(False) self._extract_report_frontpage(False) self._extract_report_keywords(False) - logger.info('Attempting to re-extract failed data from ST txts') + logger.info("Attempting to re-extract failed data from ST txts") self._extract_targets_metadata(False) self._extract_targets_frontpage(False) self._extract_targets_keywords(False) def _compute_cert_labs(self): - logger.info('Deriving information about laboratories involved in certification.') + logger.info("Deriving information about laboratories involved in certification.") certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] for cert in certs_to_process: cert.compute_heuristics_cert_lab() def _compute_cert_ids(self): - logger.info('Deriving information about sample ids from pdf scan.') + logger.info("Deriving information about sample ids from pdf scan.") certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] for cert in certs_to_process: cert.compute_heuristics_cert_id() @@ -629,7 +691,8 @@ class CCDataset(Dataset, ComplexSerializableType): def analyze_certificates(self, fresh: bool = True): if self.state.pdfs_converted is False: logger.info( - 'Attempting run analysis of txt files while not having the pdf->txt conversion done. Returning.') + "Attempting run analysis of txt files while not having the pdf->txt conversion done. Returning." + ) return self._extract_data(fresh) @@ -642,11 +705,14 @@ class CCDataset(Dataset, ComplexSerializableType): def process_maintenance_updates(self): maintained_certs: List[CommonCriteriaCert] = [x for x in self if x.maintenance_updates] - updates = list(itertools.chain.from_iterable( - [CommonCriteriaMaintenanceUpdate.get_updates_from_cc_cert(x) for x in maintained_certs])) - update_dset: CCDatasetMaintenanceUpdates = CCDatasetMaintenanceUpdates({x.dgst: x for x in updates}, - root_dir=self.certs_dir / 'maintenance', - name='Maintenance updates') + updates = list( + itertools.chain.from_iterable( + [CommonCriteriaMaintenanceUpdate.get_updates_from_cc_cert(x) for x in maintained_certs] + ) + ) + update_dset: CCDatasetMaintenanceUpdates = CCDatasetMaintenanceUpdates( + {x.dgst: x for x in updates}, root_dir=self.certs_dir / "maintenance", name="Maintenance updates" + ) update_dset.set_local_paths() update_dset.download_all_pdfs() update_dset.convert_all_pdfs() @@ -654,9 +720,9 @@ class CCDataset(Dataset, ComplexSerializableType): def generate_cert_name_keywords(self) -> Set[str]: df = self.to_pandas() - certificate_names = set(df['name']) - keywords = set(itertools.chain.from_iterable([x.lower().split(' ') for x in certificate_names])) - keywords.add('1.02.013') + certificate_names = set(df["name"]) + keywords = set(itertools.chain.from_iterable([x.lower().split(" ") for x in certificate_names])) + keywords.add("1.02.013") return {x for x in keywords if len(x) > config.minimal_token_length} @@ -664,12 +730,19 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType): """ Should be used merely for actions related to Maintenance updates: download pdfs, convert pdfs, extract data from pdfs """ + # TODO: Types - if I use dictionary in CCDataset, I can't use more specific dictionary here (otherwise the CCDataset # one would have to be a Mapping - not mutable) - certs: Dict[str, 'CommonCriteriaMaintenanceUpdate'] # type: ignore - def __init__(self, certs: Mapping[str, 'Certificate'], - root_dir: Path, name: str = 'dataset name', - description: str = 'dataset_description', state: Optional[CCDataset.DatasetInternalState] = None): + certs: Dict[str, "CommonCriteriaMaintenanceUpdate"] # type: ignore + + def __init__( + self, + certs: Mapping[str, "Certificate"], + root_dir: Path, + name: str = "dataset name", + description: str = "dataset_description", + state: Optional[CCDataset.DatasetInternalState] = None, + ): super().__init__(certs, root_dir, name, description, state) self.state.meta_sources_parsed = True @@ -689,14 +762,16 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType): @classmethod def from_json(cls, input_path: Union[str, Path]): input_path = Path(input_path) - with input_path.open('r') as handle: + with input_path.open("r") as handle: dset = json.load(handle, cls=CustomJSONDecoder) return dset def to_pandas(self): - df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=CommonCriteriaMaintenanceUpdate.pandas_columns) - df = df.set_index('dgst') - df.index.name = 'dgst' + df = pd.DataFrame( + [x.pandas_tuple for x in self.certs.values()], columns=CommonCriteriaMaintenanceUpdate.pandas_columns + ) + df = df.set_index("dgst") + df.index.name = "dgst" df.maintenance_date = pd.to_datetime(df.maintenance_date, infer_datetime_format=True) df = df.fillna(value=np.nan) @@ -706,6 +781,6 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType): @classmethod def from_web_latest(cls): with tempfile.TemporaryDirectory() as tmp_dir: - dset_path = Path(tmp_dir) / 'cc_maintenances_latest_dataset.json' + dset_path = Path(tmp_dir) / "cc_maintenances_latest_dataset.json" helpers.download_file(config.cc_maintenances_latest_snapshot, dset_path) return cls.from_json(dset_path) diff --git a/sec_certs/dataset/cpe.py b/sec_certs/dataset/cpe.py index faf82b05..6727f930 100644 --- a/sec_certs/dataset/cpe.py +++ b/sec_certs/dataset/cpe.py @@ -1,19 +1,19 @@ -from dataclasses import dataclass, field -import logging -from typing import List, Dict, Tuple, Set, Union, ClassVar import itertools +import logging import tempfile -from pathlib import Path +import xml.etree.ElementTree as ET import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import ClassVar, Dict, List, Set, Tuple, Union + +import pandas as pd import sec_certs.helpers as helpers -from sec_certs.sample.cpe import CPE from sec_certs.dataset.cve import CVEDataset +from sec_certs.sample.cpe import CPE from sec_certs.serialization.json import ComplexSerializableType, serialize -import pandas as pd -import xml.etree.ElementTree as ET - logger = logging.getLogger(__name__) @@ -23,12 +23,14 @@ class CPEDataset(ComplexSerializableType): json_path: Path cpes: Dict[str, CPE] vendor_to_versions: Dict[str, Set[str]] = field(init=False) # Look-up dict cpe_vendor: list of viable versions - vendor_version_to_cpe: Dict[Tuple[str, str], Set[CPE]] = field(init=False) # Look-up dict (cpe_vendor, cpe_version): List of viable cpe items + vendor_version_to_cpe: Dict[Tuple[str, str], Set[CPE]] = field( + init=False + ) # Look-up dict (cpe_vendor, cpe_version): List of viable cpe items title_to_cpes: Dict[str, Set[CPE]] = field(init=False) # Look-up dict title: List of cert items vendors: Set[str] = field(init=False) - cpe_xml_basename: ClassVar[str] = 'official-cpe-dictionary_v2.3.xml' - cpe_url: ClassVar[str] = 'https://nvd.nist.gov/feeds/xml/cpe/dictionary/' + cpe_xml_basename + '.zip' + cpe_xml_basename: ClassVar[str] = "official-cpe-dictionary_v2.3.xml" + cpe_url: ClassVar[str] = "https://nvd.nist.gov/feeds/xml/cpe/dictionary/" + cpe_xml_basename + ".zip" def __iter__(self): yield from self.cpes.values() @@ -44,18 +46,18 @@ class CPEDataset(ComplexSerializableType): def __contains__(self, item: CPE) -> bool: if not isinstance(item, CPE): - raise ValueError(f'{item} is not of CPE class') + raise ValueError(f"{item} is not of CPE class") return item.uri in self.cpes.keys() @property def serialized_attributes(self) -> List[str]: - return ['was_enhanced_with_vuln_cpes', 'json_path', 'cpes'] + return ["was_enhanced_with_vuln_cpes", "json_path", "cpes"] def __post_init__(self): """ Will build look-up dictionaries that are used for fast matching """ - logger.info('CPE dataset: building lookup dictionaries.') + logger.info("CPE dataset: building lookup dictionaries.") self.vendor_to_versions = {x.vendor: set() for x in self} self.vendor_version_to_cpe = dict() self.title_to_cpes = dict() @@ -75,30 +77,34 @@ class CPEDataset(ComplexSerializableType): def from_web(cls, json_path: Union[str, Path]): with tempfile.TemporaryDirectory() as tmp_dir: xml_path = Path(tmp_dir) / cls.cpe_xml_basename - zip_path = Path(tmp_dir) / (cls.cpe_xml_basename + '.zip') + zip_path = Path(tmp_dir) / (cls.cpe_xml_basename + ".zip") helpers.download_file(cls.cpe_url, zip_path) - with zipfile.ZipFile(zip_path, 'r') as zip_ref: + with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(tmp_dir) return cls.from_xml(xml_path, json_path) @classmethod def from_xml(cls, xml_path: Union[str, Path], json_path: Union[str, Path]): - logger.info('Loading CPE dataset from XML.') + logger.info("Loading CPE dataset from XML.") root = ET.parse(xml_path).getroot() dct = {} - for cpe_item in root.findall('{http://cpe.mitre.org/dictionary/2.0}cpe-item'): - found_title = cpe_item.find('{http://cpe.mitre.org/dictionary/2.0}title') + for cpe_item in root.findall("{http://cpe.mitre.org/dictionary/2.0}cpe-item"): + found_title = cpe_item.find("{http://cpe.mitre.org/dictionary/2.0}title") if found_title is None: - raise RuntimeError("Title is not found during building CPE dataset from xml - this should not be happening") + raise RuntimeError( + "Title is not found during building CPE dataset from xml - this should not be happening" + ) title = found_title.text - - found_cpe_uri = cpe_item.find('{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item') + + found_cpe_uri = cpe_item.find("{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item") if found_cpe_uri is None: - raise RuntimeError("CPE uri is not found during building CPE dataset from xml - this should not be happening") - cpe_uri = found_cpe_uri.attrib['name'] - + raise RuntimeError( + "CPE uri is not found during building CPE dataset from xml - this should not be happening" + ) + cpe_uri = found_cpe_uri.attrib["name"] + dct[cpe_uri] = CPE(cpe_uri, title) return cls(False, Path(json_path), dct) @@ -110,11 +116,11 @@ class CPEDataset(ComplexSerializableType): @classmethod def from_dict(cls, dct: Dict): - return cls(dct['was_enhanced_with_vuln_cpes'], Path('../'), dct['cpes']) + return cls(dct["was_enhanced_with_vuln_cpes"], Path("../"), dct["cpes"]) def to_pandas(self): df = pd.DataFrame([x.pandas_tuple for x in self], columns=CPE.pandas_columns) - df = df.set_index('uri') + df = df.set_index("uri") return df @serialize @@ -128,9 +134,9 @@ class CPEDataset(ComplexSerializableType): old_len = len(self.cpes) - for cpe in helpers.tqdm(all_cpes_in_cve_dset, desc='Enriching CPE dataset with new CPEs'): + for cpe in helpers.tqdm(all_cpes_in_cve_dset, desc="Enriching CPE dataset with new CPEs"): if cpe not in self: self[cpe.uri] = cpe - logger.info(f'Enriched the CPE dataset with {len(self.cpes) - old_len} new CPE records.') + logger.info(f"Enriched the CPE dataset with {len(self.cpes) - old_len} new CPE records.") self.was_enhanced_with_vuln_cpes = True diff --git a/sec_certs/dataset/cve.py b/sec_certs/dataset/cve.py index 47eab91b..6e662ac6 100644 --- a/sec_certs/dataset/cve.py +++ b/sec_certs/dataset/cve.py @@ -1,24 +1,24 @@ -import itertools -from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple, Union, Final, Set import datetime -from pathlib import Path -import tempfile -import zipfile -import logging import glob +import itertools import json +import logging import shutil +import tempfile +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Final, List, Optional, Set, Union import pandas as pd -from sec_certs.parallel_processing import process_parallel import sec_certs.constants as constants import sec_certs.helpers as helpers -from sec_certs.sample.cve import CVE +from sec_certs.config.configuration import config +from sec_certs.parallel_processing import process_parallel from sec_certs.sample.cpe import CPE +from sec_certs.sample.cve import CVE from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder -from sec_certs.config.configuration import config logger = logging.getLogger(__name__) @@ -27,12 +27,12 @@ logger = logging.getLogger(__name__) class CVEDataset(ComplexSerializableType): cves: Dict[str, CVE] cpe_to_cve_ids_lookup: Dict[str, List[str]] = field(init=False) - cve_url: Final[str] = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-' - cpe_match_feed_url: Final[str] = 'https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip' + cve_url: Final[str] = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" + cpe_match_feed_url: Final[str] = "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip" @property def serialized_attributes(self) -> List[str]: - return ['cves'] + return ["cves"] def __iter__(self): yield from self.cves.values() @@ -61,15 +61,17 @@ class CVEDataset(ComplexSerializableType): self.cpe_to_cve_ids_lookup = dict() self.cves = {x.cve_id.upper(): x for x in self} - logger.info('Getting CPE matching dictionary from NIST.gov') + logger.info("Getting CPE matching dictionary from NIST.gov") if use_nist_mapping: matching_dict = self.get_nist_cpe_matching_dict(nist_matching_filepath) - for cve in helpers.tqdm(self, desc='Building-up lookup dictionaries for fast CVE matching'): + for cve in helpers.tqdm(self, desc="Building-up lookup dictionaries for fast CVE matching"): # See note above, we use matching_dict.get(cpe, []) instead of matching_dict[cpe] as would be expected if use_nist_mapping: - vulnerable_configurations = itertools.chain.from_iterable([matching_dict.get(cpe, []) for cpe in cve.vulnerable_cpes]) + vulnerable_configurations = itertools.chain.from_iterable( + [matching_dict.get(cpe, []) for cpe in cve.vulnerable_cpes] + ) else: vulnerable_configurations = cve.vulnerable_cpes for cpe in vulnerable_configurations: @@ -84,40 +86,45 @@ class CVEDataset(ComplexSerializableType): if not output_path.exists: output_path.mkdir() - urls = [cls.cve_url + str(x) + '.json.zip' for x in range(start_year, end_year + 1)] + urls = [cls.cve_url + str(x) + ".json.zip" for x in range(start_year, end_year + 1)] - logger.info(f'Identified {len(urls)} CVE files to fetch from nist.gov. Downloading them into {output_path}') + logger.info(f"Identified {len(urls)} CVE files to fetch from nist.gov. Downloading them into {output_path}") with tempfile.TemporaryDirectory() as tmp_dir: - outpaths = [Path(tmp_dir) / Path(x).name.rstrip('.zip') for x in urls] - responses = list(zip(*helpers.download_parallel(list(zip(urls, outpaths)), num_threads=config.n_threads)))[1] + outpaths = [Path(tmp_dir) / Path(x).name.rstrip(".zip") for x in urls] + responses = list(zip(*helpers.download_parallel(list(zip(urls, outpaths)), num_threads=config.n_threads)))[ + 1 + ] for o, u, r in zip(outpaths, urls, responses): if r == constants.RESPONSE_OK: - with zipfile.ZipFile(o, 'r') as zip_handle: + with zipfile.ZipFile(o, "r") as zip_handle: zip_handle.extractall(output_path) else: - logger.info(f'Failed to download from {u}, got status code {r}') + logger.info(f"Failed to download from {u}, got status code {r}") @classmethod - def from_nist_json(cls, input_path: str) -> 'CVEDataset': - with Path(input_path).open('r') as handle: + def from_nist_json(cls, input_path: str) -> "CVEDataset": + with Path(input_path).open("r") as handle: data = json.load(handle) - cves = [CVE.from_nist_dict(x) for x in data['CVE_Items']] + cves = [CVE.from_nist_dict(x) for x in data["CVE_Items"]] return cls({x.cve_id: x for x in cves}) @classmethod - def from_web(cls, - start_year: int = 2002, - end_year: int = datetime.datetime.now().year): - logger.info(f'Building CVE dataset from nist.gov website.') + def from_web(cls, start_year: int = 2002, end_year: int = datetime.datetime.now().year): + logger.info("Building CVE dataset from nist.gov website.") with tempfile.TemporaryDirectory() as tmp_dir: cls.download_cves(tmp_dir, start_year, end_year) - json_files = glob.glob(tmp_dir + '/*.json') + json_files = glob.glob(tmp_dir + "/*.json") all_cves = dict() - logger.info(f'Downloaded required resources. Building CVEDataset from jsons.') - results = process_parallel(cls.from_nist_json, json_files, config.n_threads, use_threading=False, - progress_bar_desc='Building CVEDataset from jsons') + logger.info("Downloaded required resources. Building CVEDataset from jsons.") + results = process_parallel( + cls.from_nist_json, + json_files, + config.n_threads, + use_threading=False, + progress_bar_desc="Building CVEDataset from jsons", + ) for r in results: all_cves.update(r.cves) @@ -125,13 +132,15 @@ class CVEDataset(ComplexSerializableType): def to_json(self, output_path: Optional[Union[str, Path]] = None): if output_path is None: - raise RuntimeError(f"You tried to serialize an object ({type(self)}) that does not have implicit json path. Please provide json_path.") - with Path(output_path).open('w') as handle: + raise RuntimeError( + f"You tried to serialize an object ({type(self)}) that does not have implicit json path. Please provide json_path." + ) + with Path(output_path).open("w") as handle: json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False) @classmethod def from_json(cls, input_path: Union[str, Path]): - with Path(input_path).open('r') as handle: + with Path(input_path).open("r") as handle: dset = json.load(handle, cls=CustomJSONDecoder) return dset @@ -152,59 +161,61 @@ class CVEDataset(ComplexSerializableType): for cve in self: n_cpes_orig = len(cve.vulnerable_cpes) cve.vulnerable_cpes = list(filter(lambda x: x in relevant_cpes, cve.vulnerable_cpes)) - total_deleted_cpes += (n_cpes_orig - len(cve.vulnerable_cpes)) + total_deleted_cpes += n_cpes_orig - len(cve.vulnerable_cpes) if not cve.vulnerable_cpes: cve_ids_to_delete.append(cve.cve_id) for cve_id in cve_ids_to_delete: del self.cves[cve_id] - logger.info(f'Totally deleted {total_deleted_cpes} irrelevant CPEs and {len(cve_ids_to_delete)} CVEs from CVEDataset.') + logger.info( + f"Totally deleted {total_deleted_cpes} irrelevant CPEs and {len(cve_ids_to_delete)} CVEs from CVEDataset." + ) def to_pandas(self) -> pd.DataFrame: df = pd.DataFrame([x.pandas_tuple for x in self], columns=CVE.pandas_columns) - return df.set_index('cve_id') + return df.set_index("cve_id") def get_nist_cpe_matching_dict(self, input_filepath: Optional[Path]): def parse_key_cpe(field: Dict) -> CPE: start_version = None - if 'versionStartIncluding' in field: - start_version = ('including', field['versionStartIncluding']) - elif 'versionStartExcluding' in field: - start_version = ('excluding', field['versionStartExcluding']) + if "versionStartIncluding" in field: + start_version = ("including", field["versionStartIncluding"]) + elif "versionStartExcluding" in field: + start_version = ("excluding", field["versionStartExcluding"]) end_version = None - if 'versionEndIncluding' in field: - end_version = ('including', field['versionEndIncluding']) - elif 'versionEndExcluding' in field: - end_version = ('excluding', field['versionEndExcluding']) + if "versionEndIncluding" in field: + end_version = ("including", field["versionEndIncluding"]) + elif "versionEndExcluding" in field: + end_version = ("excluding", field["versionEndExcluding"]) - return CPE(field['cpe23Uri'], start_version=start_version, end_version=end_version) + return CPE(field["cpe23Uri"], start_version=start_version, end_version=end_version) def parse_values_cpe(field: Dict) -> List[CPE]: - return [CPE(x['cpe23Uri']) for x in field['cpe_name']] - - logger.debug('Attempting to get NIST mapping file.') + return [CPE(x["cpe23Uri"]) for x in field["cpe_name"]] + + logger.debug("Attempting to get NIST mapping file.") if not input_filepath or not input_filepath.is_file(): - logger.debug('NIST mapping file not available, going to download.') + logger.debug("NIST mapping file not available, going to download.") with tempfile.TemporaryDirectory() as tmp_dir: filename = Path(self.cpe_match_feed_url).name download_path = Path(tmp_dir) / filename - unzipped_path = Path(tmp_dir) / filename.rstrip('.zip') + unzipped_path = Path(tmp_dir) / filename.rstrip(".zip") helpers.download_file(self.cpe_match_feed_url, download_path) - with zipfile.ZipFile(download_path, 'r') as zip_handle: + with zipfile.ZipFile(download_path, "r") as zip_handle: zip_handle.extractall(tmp_dir) - with unzipped_path.open('r') as handle: + with unzipped_path.open("r") as handle: match_data = json.load(handle) if input_filepath: - logger.debug(f'Copying attained NIST mapping file to {input_filepath}') + logger.debug(f"Copying attained NIST mapping file to {input_filepath}") shutil.move(str(unzipped_path), str(input_filepath)) else: - with input_filepath.open('r') as handle: + with input_filepath.open("r") as handle: match_data = json.load(handle) mapping_dict = dict() - for match in helpers.tqdm(match_data['matches'], desc='parsing cpe matching (by NIST) dictionary'): + for match in helpers.tqdm(match_data["matches"], desc="parsing cpe matching (by NIST) dictionary"): key = parse_key_cpe(match) value = parse_values_cpe(match) mapping_dict[key] = value if value else [key] diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py index be06416c..eeae0601 100644 --- a/sec_certs/dataset/dataset.py +++ b/sec_certs/dataset/dataset.py @@ -1,36 +1,40 @@ -from datetime import datetime -import logging -from typing import Dict, Collection, Optional, Set, Union, List, Tuple, Mapping - +import itertools import json +import logging from abc import ABC, abstractmethod +from datetime import datetime from pathlib import Path -import itertools +from typing import Collection, Dict, List, Mapping, Optional, Set, Tuple, Type, TypeVar, Union import requests -import sec_certs.helpers as helpers import sec_certs.constants as constants +import sec_certs.helpers as helpers import sec_certs.parallel_processing as cert_processing -from sec_certs.sample.cpe import CPE - -from sec_certs.sample.certificate import Certificate -from sec_certs.serialization.json import ComplexSerializableType from sec_certs.config.configuration import config -from sec_certs.serialization.json import serialize from sec_certs.dataset.cpe import CPEDataset from sec_certs.dataset.cve import CVEDataset from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.sample.certificate import Certificate +from sec_certs.sample.cpe import CPE +from sec_certs.serialization.json import ComplexSerializableType, serialize logger = logging.getLogger(__name__) +T = TypeVar("T") + class Dataset(ABC): - def __init__(self, certs: Mapping[str, 'Certificate'], root_dir: Path, name: str = 'dataset name', - description: str = 'dataset_description'): + def __init__( + self, + certs: Mapping[str, "Certificate"], + root_dir: Path, + name: str = "dataset name", + description: str = "dataset_description", + ): self._root_dir = root_dir self.timestamp = datetime.now() - self.sha256_digest = 'not implemented' + self.sha256_digest = "not implemented" self.name = name self.description = description self.certs = certs @@ -47,27 +51,27 @@ class Dataset(ABC): @property def web_dir(self) -> Path: - return self.root_dir / 'web' + return self.root_dir / "web" @property def auxillary_datasets_dir(self) -> Path: - return self.root_dir / 'auxillary_datasets' + return self.root_dir / "auxillary_datasets" @property def cpe_dataset_path(self) -> Path: - return self.auxillary_datasets_dir / 'cpe_dataset.json' + return self.auxillary_datasets_dir / "cpe_dataset.json" @property def cve_dataset_path(self) -> Path: - return self.auxillary_datasets_dir / 'cve_dataset.json' + return self.auxillary_datasets_dir / "cve_dataset.json" @property def nist_cve_cpe_matching_dset_path(self) -> Path: - return self.auxillary_datasets_dir / 'nvdcpematch-1.0.json' + return self.auxillary_datasets_dir / "nvdcpematch-1.0.json" @property def json_path(self) -> Path: - return self.root_dir / (self.name + '.json') + return self.root_dir / (self.name + ".json") def __contains__(self, item): if not issubclass(type(item), Certificate): @@ -80,8 +84,8 @@ class Dataset(ABC): def __getitem__(self, item: str): return self.certs.__getitem__(item.lower()) - def __setitem__(self, key: str, value: 'Certificate'): - self.certs.__setitem__(key.lower(), value) # type: ignore + def __setitem__(self, key: str, value: "Certificate"): + self.certs.__setitem__(key.lower(), value) # type: ignore def __len__(self) -> int: return len(self.certs) @@ -92,65 +96,70 @@ class Dataset(ABC): return self.certs == other.certs def __str__(self) -> str: - return str(type(self).__name__) + ':' + self.name + ', ' + str(len(self)) + ' certificates' + return str(type(self).__name__) + ":" + self.name + ", " + str(len(self)) + " certificates" def to_dict(self): - return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest, - 'name': self.name, 'description': self.description, - 'n_certs': len(self), 'certs': list(self.certs.values())} + return { + "timestamp": self.timestamp, + "sha256_digest": self.sha256_digest, + "name": self.name, + "description": self.description, + "n_certs": len(self), + "certs": list(self.certs.values()), + } @classmethod def from_dict(cls, dct: Dict): - certs = {x.dgst: x for x in dct['certs']} - dset = cls(certs, Path('../'), dct['name'], dct['description']) - if len(dset) != (claimed := dct['n_certs']): + certs = {x.dgst: x for x in dct["certs"]} + dset = cls(certs, Path("../"), dct["name"], dct["description"]) + if len(dset) != (claimed := dct["n_certs"]): logger.error( - f'The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed}).') + f"The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed})." + ) return dset @classmethod - def from_json(cls, input_path: Union[str, Path]): + def from_json(cls: Type[T], input_path: Union[str, Path]) -> T: dset = ComplexSerializableType.from_json(input_path) dset.root_dir = Path(input_path).parent.absolute() dset.set_local_paths() return dset def set_local_paths(self): - raise NotImplementedError('Not meant to be implemented by the base class.') + raise NotImplementedError("Not meant to be implemented by the base class.") @abstractmethod def get_certs_from_web(self): - raise NotImplementedError('Not meant to be implemented by the base class.') + raise NotImplementedError("Not meant to be implemented by the base class.") @abstractmethod def convert_all_pdfs(self): - raise NotImplementedError('Not meant to be implemented by the base class.') + raise NotImplementedError("Not meant to be implemented by the base class.") @abstractmethod def download_all_pdfs(self, cert_ids: Optional[Set[str]] = None): - raise NotImplementedError('Not meant to be implemented by the base class.') + raise NotImplementedError("Not meant to be implemented by the base class.") @staticmethod def _download_parallel(urls: Collection[str], paths: Collection[Path], prune_corrupted: bool = True): - exit_codes = cert_processing.process_parallel(helpers.download_file, - list(zip(urls, paths)), - config.n_threads, - unpack=True) + exit_codes = cert_processing.process_parallel( + helpers.download_file, list(zip(urls, paths)), config.n_threads, unpack=True + ) n_successful = len([e for e in exit_codes if e == requests.codes.ok]) - logger.info(f'Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.') + logger.info(f"Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.") for url, e in zip(urls, exit_codes): if e != requests.codes.ok: - logger.error(f'Failed to download {url}, exit code: {e}') + logger.error(f"Failed to download {url}, exit code: {e}") if prune_corrupted is True: for p in paths: if p.exists() and p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE: - logger.error(f'Corrupted file at: {p}') + logger.error(f"Corrupted file at: {p}") p.unlink() def _prepare_cpe_dataset(self, download_fresh_cpes: bool = False): - logger.info('Preparing CPE dataset.') + logger.info("Preparing CPE dataset.") if not self.auxillary_datasets_dir.exists(): self.auxillary_datasets_dir.mkdir(parents=True) @@ -162,8 +171,10 @@ class Dataset(ABC): return cpe_dataset - def _prepare_cve_dataset(self, download_fresh_cves: bool = False, use_nist_cpe_matching_dict: bool = True) -> CVEDataset: - logger.info('Preparing CVE dataset.') + def _prepare_cve_dataset( + self, download_fresh_cves: bool = False, use_nist_cpe_matching_dict: bool = True + ) -> CVEDataset: + logger.info("Preparing CVE dataset.") if not self.auxillary_datasets_dir.exists(): self.auxillary_datasets_dir.mkdir(parents=True) @@ -177,7 +188,7 @@ class Dataset(ABC): return cve_dataset def _compute_candidate_versions(self): - logger.info('Computing heuristics: possible product versions in sample name') + logger.info("Computing heuristics: possible product versions in sample name") for cert in self: cert.compute_heuristics_version() @@ -186,13 +197,22 @@ class Dataset(ABC): """ Filters out very weak CPE matches that don't improve our database. """ - if cpe.title and (cpe.version == '-' or cpe.version == '*') and not any(char.isdigit() for char in cpe.title): + if ( + cpe.title + and (cpe.version == "-" or cpe.version == "*") + and not any(char.isdigit() for char in cpe.title) + ): return False - elif not cpe.title and cpe.item_name and (cpe.version == '-' or cpe.version == '*') and not any(char.isdigit() for char in cpe.item_name): + elif ( + not cpe.title + and cpe.item_name + and (cpe.version == "-" or cpe.version == "*") + and not any(char.isdigit() for char in cpe.item_name) + ): return False return True - logger.info('Computing heuristics: Finding CPE matches for certificates') + logger.info("Computing heuristics: Finding CPE matches for certificates") cpe_dset = self._prepare_cpe_dataset(download_fresh_cpes) if not cpe_dset.was_enhanced_with_vuln_cpes: cve_dset = self._prepare_cve_dataset(False) @@ -201,7 +221,7 @@ class Dataset(ABC): clf = CPEClassifier(config.cpe_matching_threshold, config.cpe_n_max_matches) clf.fit([x for x in cpe_dset if filter_condition(x)]) - for cert in helpers.tqdm(self, desc='Predicting CPE matches with the classifier'): + for cert in helpers.tqdm(self, desc="Predicting CPE matches with the classifier"): cert.compute_heuristics_cpe_match(clf) return clf, cpe_dset @@ -214,51 +234,52 @@ class Dataset(ABC): def to_label_studio_json(self, output_path: Union[str, Path]): lst = [] for cert in [x for x in self if x.heuristics.cpe_matches]: - dct = {'text': cert.label_studio_title} + dct = {"text": cert.label_studio_title} candidates = [x[1].title for x in cert.heuristics.cpe_matches] - candidates += ['No good match'] * (config.cc_cpe_max_matches - len(candidates)) - options = ['option_' + str(x) for x in range(1, 21)] + candidates += ["No good match"] * (config.cc_cpe_max_matches - len(candidates)) + options = ["option_" + str(x) for x in range(1, 21)] dct.update({o: c for o, c in zip(options, candidates)}) lst.append(dct) - with Path(output_path).open('w') as handle: + with Path(output_path).open("w") as handle: json.dump(lst, handle, indent=4) @serialize def load_label_studio_labels(self, input_path: Union[str, Path]): - with Path(input_path).open('r') as handle: + with Path(input_path).open("r") as handle: data = json.load(handle) cpe_dset = self._prepare_cpe_dataset() - logger.info('Translating label studio matches into their CPE representations and assigning to certificates.') - for annotation in helpers.tqdm([x for x in data if 'verified_cpe_match' in x], desc='Translating label studio matches'): - match_keys = annotation['verified_cpe_match'] - match_keys = [match_keys] if isinstance(match_keys, str) else match_keys['choices'] - match_keys = [x.lstrip('$') for x in match_keys] - predicted_annotations = [annotation[x] for x in match_keys if annotation[x] != 'No good match'] + logger.info("Translating label studio matches into their CPE representations and assigning to certificates.") + for annotation in helpers.tqdm( + [x for x in data if "verified_cpe_match" in x], desc="Translating label studio matches" + ): + match_keys = annotation["verified_cpe_match"] + match_keys = [match_keys] if isinstance(match_keys, str) else match_keys["choices"] + match_keys = [x.lstrip("$") for x in match_keys] + predicted_annotations = [annotation[x] for x in match_keys if annotation[x] != "No good match"] cpes: Set[Optional[CPE]] = set() for x in predicted_annotations: if x not in cpe_dset.title_to_cpes: - print(f'Error: {x} not in dataset') + print(f"Error: {x} not in dataset") else: to_update = cpe_dset.title_to_cpes[x] if to_update and not cpes: cpes = to_update elif to_update and cpes: - # TODO: This was here like cpes = cpes.update(to_update), but update() does not return anything. - # Did you try to hack something using that or was that just a typo? + # TODO: This was here like cpes = cpes.update(to_update), but update() does not return anything. + # Did you try to hack something using that or was that just a typo? cpes.update(to_update) - # cpes = set(itertools.chain.from_iterable([cpe_dset.title_to_cpes.get(x, []) for x in predicted_annotations])) # distinguish between FIPS and CC - if '\n' in annotation['text']: - cert_name = annotation['text'].split('\nModule name: ')[1].split('\n')[0] + if "\n" in annotation["text"]: + cert_name = annotation["text"].split("\nModule name: ")[1].split("\n")[0] else: - cert_name = annotation['text'] + cert_name = annotation["text"] certs = self.get_certs_from_name(cert_name) @@ -266,7 +287,7 @@ class Dataset(ABC): c.heuristics.verified_cpe_matches = {x.uri for x in cpes if x is not None} if cpes else None def get_certs_from_name(self, name: str) -> List[Certificate]: - raise NotImplementedError('Not meant to be implemented by the base class.') + raise NotImplementedError("Not meant to be implemented by the base class.") def enrich_automated_cpes_with_manual_labels(self): """ @@ -276,27 +297,32 @@ class Dataset(ABC): if not cert.heuristics.cpe_matches and cert.heuristics.verified_cpe_matches: cert.heuristics.cpe_matches = cert.heuristics.verified_cpe_matches elif cert.heuristics.cpe_matches and cert.heuristics.verified_cpe_matches: - cert.heuristics.cpe_matches = set(cert.heuristics.cpe_matches).union(set(cert.heuristics.verified_cpe_matches)) + cert.heuristics.cpe_matches = set(cert.heuristics.cpe_matches).union( + set(cert.heuristics.verified_cpe_matches) + ) @serialize def compute_related_cves(self, download_fresh_cves: bool = False, use_nist_cpe_matching_dict: bool = True): - logger.info('Retrieving related CVEs to verified CPE matches') + logger.info("Retrieving related CVEs to verified CPE matches") cve_dset = self._prepare_cve_dataset(download_fresh_cves, use_nist_cpe_matching_dict) self.enrich_automated_cpes_with_manual_labels() cpe_rich_certs = [x for x in self if x.heuristics.cpe_matches] if not cpe_rich_certs: - logger.error('No certificates with verified CPE match detected. You must run dset.manually_verify_cpe_matches() first. Returning.') + logger.error( + "No certificates with verified CPE match detected. You must run dset.manually_verify_cpe_matches() first. Returning." + ) return relevant_cpes = set(itertools.chain.from_iterable([x.heuristics.cpe_matches for x in cpe_rich_certs])) cve_dset.filter_related_cpes(relevant_cpes) - for cert in helpers.tqdm(cpe_rich_certs, desc='Computing related CVES'): + for cert in helpers.tqdm(cpe_rich_certs, desc="Computing related CVES"): cert.compute_heuristics_related_cves(cve_dset) n_vulnerable = len([x for x in cpe_rich_certs if x.heuristics.related_cves]) - n_vulnerabilities = sum( - [len(x.heuristics.related_cves) for x in cpe_rich_certs if x.heuristics.related_cves]) - logger.info(f'In total, we identified {n_vulnerabilities} vulnerabilities in {n_vulnerable} vulnerable certificates.') + n_vulnerabilities = sum([len(x.heuristics.related_cves) for x in cpe_rich_certs if x.heuristics.related_cves]) + logger.info( + f"In total, we identified {n_vulnerabilities} vulnerabilities in {n_vulnerable} vulnerable certificates." + ) diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py index f252bffc..d1ce2dea 100644 --- a/sec_certs/dataset/fips.py +++ b/sec_certs/dataset/fips.py @@ -1,22 +1,23 @@ import datetime -import tempfile import logging import os +import tempfile from itertools import groupby from pathlib import Path +from typing import Dict, List, Mapping, Optional, Set, Tuple -from typing import Set, Tuple, List, Dict, Optional, Mapping from bs4 import BeautifulSoup, NavigableString from graphviz import Digraph -from sec_certs import constants as constants, parallel_processing as cert_processing, helpers as helpers +from sec_certs import constants as constants +from sec_certs import helpers as helpers +from sec_certs import parallel_processing as cert_processing from sec_certs.config.configuration import config -from sec_certs.sample.certificate import Certificate -from sec_certs.dataset.dataset import Dataset, logger +from sec_certs.dataset.dataset import Dataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.serialization.json import ComplexSerializableType, serialize +from sec_certs.sample.certificate import Certificate from sec_certs.sample.fips import FIPSCertificate - +from sec_certs.serialization.json import ComplexSerializableType, serialize logger = logging.getLogger(__name__) @@ -25,7 +26,11 @@ class FIPSDataset(Dataset, ComplexSerializableType): certs: Dict[str, FIPSCertificate] def __init__( - self, certs: Mapping[str, 'Certificate'], root_dir: Path, name: str = "dataset name", description: str = "dataset_description" + self, + certs: Mapping[str, "Certificate"], + root_dir: Path, + name: str = "dataset name", + description: str = "dataset_description", ): super().__init__(certs, root_dir, name, description) self.keywords: Dict[str, Dict] = {} @@ -81,7 +86,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): [cert for cert in self.certs.values() if not cert.pdf_scan.keywords or redo], config.n_threads, use_threading=False, - progress_bar_desc="Scanning PDF files" + progress_bar_desc="Scanning PDF files", ) for keyword, cert in keywords: self.certs[cert.dgst].pdf_scan.keywords = keyword @@ -92,7 +97,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): # if the pdf has not been processed, no matching can be done if not cert.pdf_scan.keywords or not cert.state.txt_state: continue - + output[cert.dgst] = FIPSCertificate.match_web_algs_to_pdf(cert) cert.heuristics.unmatched_algs = output[cert.dgst] @@ -105,7 +110,8 @@ class FIPSDataset(Dataset, ComplexSerializableType): if cert_ids is None: raise RuntimeError("You need to provide cert ids to FIPS download PDFs functionality.") for cert_id in cert_ids: - if not (self.policies_dir / f"{cert_id}.pdf").exists() or (cert_id in self.certs and not self.certs[cert_id].state.txt_state + if not (self.policies_dir / f"{cert_id}.pdf").exists() or ( + cert_id in self.certs and not self.certs[cert_id].state.txt_state ): sp_urls.append( f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf" @@ -113,7 +119,10 @@ class FIPSDataset(Dataset, ComplexSerializableType): sp_paths.append(self.policies_dir / f"{cert_id}.pdf") logger.info(f"downloading {len(sp_urls)} module pdf files") cert_processing.process_parallel( - FIPSCertificate.download_security_policy, list(zip(sp_urls, sp_paths)), config.n_threads, progress_bar_desc="Downloading PDF files" + FIPSCertificate.download_security_policy, + list(zip(sp_urls, sp_paths)), + config.n_threads, + progress_bar_desc="Downloading PDF files", ) self.new_files += len(sp_urls) @@ -131,31 +140,45 @@ class FIPSDataset(Dataset, ComplexSerializableType): logger.info(f"downloading {len(html_urls)} module html files") failed = cert_processing.process_parallel( - FIPSCertificate.download_html_page, list(zip(html_urls, html_paths)), config.n_threads, progress_bar_desc="Downloading HTML files" + FIPSCertificate.download_html_page, + list(zip(html_urls, html_paths)), + config.n_threads, + progress_bar_desc="Downloading HTML files", ) failed = [c for c in failed if c] self.new_files += len(html_urls) if len(failed) != 0: logger.info(f"Download failed for {len(failed)} files. Retrying...") - cert_processing.process_parallel(FIPSCertificate.download_html_page, failed, config.n_threads, progress_bar_desc="Downloading HTML files again") + cert_processing.process_parallel( + FIPSCertificate.download_html_page, + failed, + config.n_threads, + progress_bar_desc="Downloading HTML files again", + ) return new_files @serialize def convert_all_pdfs(self): - logger.info('Converting FIPS sample reports to .txt') + logger.info("Converting FIPS sample reports to .txt") tuples = [ (cert, self.policies_dir / f"{cert.cert_id}.pdf", self.policies_dir / f"{cert.cert_id}.pdf.txt") for cert in self.certs.values() if not cert.state.txt_state and (self.policies_dir / f"{cert.cert_id}.pdf").exists() ] - cert_processing.process_parallel(FIPSCertificate.convert_pdf_file, tuples, config.n_threads, progress_bar_desc="Converting to txt") + cert_processing.process_parallel( + FIPSCertificate.convert_pdf_file, tuples, config.n_threads, progress_bar_desc="Converting to txt" + ) def prepare_dataset(self, test: Optional[Path] = None, update: bool = False) -> Set[str]: if test: html_files = [test] else: - html_files = [Path("fips_modules_active.html"), Path("fips_modules_historical.html"), Path("fips_modules_revoked.html")] + html_files = [ + Path("fips_modules_active.html"), + Path("fips_modules_historical.html"), + Path("fips_modules_revoked.html"), + ] helpers.download_file( "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0", Path(self.web_dir / "fips_modules_active.html"), @@ -173,7 +196,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): cert_ids: Set[str] = set() for f in html_files: cert_ids |= self._get_certificates_from_html(self.web_dir / f, update) - + return cert_ids def download_neccessary_files(self, cert_ids: Set[str]): @@ -183,18 +206,18 @@ class FIPSDataset(Dataset, ComplexSerializableType): def _get_certificates_from_html(self, html_file: Path, update: bool = False) -> Set[str]: logger.info(f"Getting certificate ids from {html_file}") with open(html_file, "r", encoding="utf-8") as handle: - html = BeautifulSoup(handle.read(), 'html5lib') + html = BeautifulSoup(handle.read(), "html5lib") table = [x for x in html.find(id="searchResultsTable").tbody.contents if x != "\n"] entries: Set[str] = set() - + for entry in table: if isinstance(entry, NavigableString): continue cert_id = entry.find("a").text if cert_id not in entries: entries.add(cert_id) - + return entries @serialize @@ -218,12 +241,12 @@ class FIPSDataset(Dataset, ComplexSerializableType): @classmethod def from_web_latest(cls): with tempfile.TemporaryDirectory() as tmp_dir: - dset_path = Path(tmp_dir) / 'fips_latest_dataset.json' - logger.info('Downloading the latest FIPS dataset.') + dset_path = Path(tmp_dir) / "fips_latest_dataset.json" + logger.info("Downloading the latest FIPS dataset.") helpers.download_file(config.fips_latest_snapshot, dset_path) dset: FIPSDataset = cls.from_json(dset_path) - logger.info('The dataset with %s certs and %s algorithms.', len(dset), len(dset.algorithms)) - logger.info('The dataset does not contain the results of the dependency analysis - calculating them now...') + logger.info("The dataset with %s certs and %s algorithms.", len(dset), len(dset.algorithms)) + logger.info("The dataset does not contain the results of the dependency analysis - calculating them now...") dset.finalize_results() return dset @@ -231,17 +254,17 @@ class FIPSDataset(Dataset, ComplexSerializableType): cert: FIPSCertificate for cert in self.certs.values(): cert.set_local_paths(self.policies_dir, self.web_dir, self.fragments_dir) - + @serialize def get_certs_from_web( self, test: Optional[Path] = None, no_download_algorithms: bool = False, update: bool = False, - redo_web_scan = False + redo_web_scan=False, ): """Downloads HTML search pages, parses them, populates the dataset, - and performs `web-scan` - extracting information from CMVP pages for + and performs `web-scan` - extracting information from CMVP pages for each certificate. Args: @@ -263,25 +286,26 @@ class FIPSDataset(Dataset, ComplexSerializableType): self.download_neccessary_files(cert_ids) if not no_download_algorithms: - aset = FIPSAlgorithmDataset({}, Path(self.root_dir / 'web' / 'algorithms'), 'algorithms', 'sample algs') + aset = FIPSAlgorithmDataset({}, Path(self.root_dir / "web" / "algorithms"), "algorithms", "sample algs") aset.get_certs_from_web() - logger.info(f'Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.') + logger.info(f"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.") self.algorithms = aset - + self.web_scan(cert_ids, redo=redo_web_scan) @serialize def deprocess(self): - #TODO - logger.info("Removing 'heuristics' field. This dataset can be used to be uploaded and later downloaded using latest_snapshot() or something") + # TODO + logger.info( + "Removing 'heuristics' field. This dataset can be used to be uploaded and later downloaded using latest_snapshot() or something" + ) cert: FIPSCertificate for cert in self.certs.values(): cert.heuristics = FIPSCertificate.FIPSHeuristics(None, {}, [], 0) self.match_algs() - @serialize def extract_certs_from_tables(self, high_precision: bool) -> List[Path]: """ @@ -299,12 +323,12 @@ class FIPSDataset(Dataset, ComplexSerializableType): config.n_threads // 4, # tabula already processes by parallel, so # it's counterproductive to use all threads use_threading=False, - progress_bar_desc="Searching tables" + progress_bar_desc="Searching tables", ) not_decoded = [cert.state.sp_path for done, cert, _ in result if done is False] for state, cert, algorithms in result: - certificate = self.certs[cert.dgst] + certificate = self.certs[cert.dgst] certificate.state.tables_done = state certificate.pdf_scan.algorithms += algorithms return not_decoded @@ -333,18 +357,25 @@ class FIPSDataset(Dataset, ComplexSerializableType): def _compare_certs(self, current_certificate: "FIPSCertificate", other_id: str): other_cert = self.certs[other_id] - if current_certificate.web_scan.date_validation is None \ - or other_cert is None or other_cert.web_scan.date_validation is None: - raise RuntimeError("Building of the dataset probably failed - this should not be happening.") - + if ( + current_certificate.web_scan.date_validation is None + or other_cert is None + or other_cert.web_scan.date_validation is None + ): + raise RuntimeError("Building of the dataset probably failed - this should not be happening.") + cert_first = current_certificate.web_scan.date_validation[0] cert_last = current_certificate.web_scan.date_validation[-1] conn_first = other_cert.web_scan.date_validation[0] conn_last = other_cert.web_scan.date_validation[-1] - - if not isinstance(cert_first, datetime.date) or not isinstance(cert_last, datetime.date)\ - or not isinstance(conn_first, datetime.date) or not isinstance(conn_last, datetime.date): - raise RuntimeError("Dataset was probably not built correctly - this should not be happening.") + + if ( + not isinstance(cert_first, datetime.date) + or not isinstance(cert_last, datetime.date) + or not isinstance(conn_first, datetime.date) + or not isinstance(conn_last, datetime.date) + ): + raise RuntimeError("Dataset was probably not built correctly - this should not be happening.") return ( cert_first.year - conn_first.year > config.year_difference_between_validations @@ -373,10 +404,10 @@ class FIPSDataset(Dataset, ComplexSerializableType): processed_cert, cert_candidate ): return False - + if self.algorithms is None: raise RuntimeError("Dataset was probably not built correctly - this should not be happening.") - + if cert_candidate not in self.algorithms.certs: return True @@ -447,7 +478,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): def _highlight_vendor_in_dot(self, dot: Digraph, current_key: str, highlighted_vendor: str): current_cert = self.certs[current_key] - + if current_cert.web_scan.vendor != highlighted_vendor: return @@ -467,11 +498,9 @@ class FIPSDataset(Dataset, ComplexSerializableType): self._highlight_vendor_in_dot(dot, current_key, highlighted_vendor) dot.node( current_key, - label=current_key - + " " - + current_cert.web_scan.vendor if current_cert.web_scan.vendor is not None else "" - + " " - + (current_cert.web_scan.module_name if current_cert.web_scan.module_name else ""), + label=current_key + " " + current_cert.web_scan.vendor + if current_cert.web_scan.vendor is not None + else "" + " " + (current_cert.web_scan.module_name if current_cert.web_scan.module_name else ""), ) def _get_processed_list(self, connection_list: str, key: str): @@ -507,7 +536,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): for key in self.certs: cert = self.certs[key] - + if key == "Not found" or not cert.state.file_status: continue @@ -521,15 +550,14 @@ class FIPSDataset(Dataset, ComplexSerializableType): self._highlight_vendor_in_dot(dot, key, highlighted_vendor) single_dot.node( key, - label=key - + "\r\n" - + cert.web_scan.vendor if cert.web_scan.vendor is not None else "" - + ("\r\n" + cert.web_scan.module_name if cert.web_scan.module_name else ""), + label=key + "\r\n" + cert.web_scan.vendor + if cert.web_scan.vendor is not None + else "" + ("\r\n" + cert.web_scan.module_name if cert.web_scan.module_name else ""), ) for key in self.certs: cert = self.certs[key] - + if key == "Not found" or not cert.state.file_status: continue processed = self._get_processed_list(connection_list, key) @@ -544,9 +572,15 @@ class FIPSDataset(Dataset, ComplexSerializableType): single_dot.render(self.root_dir / (str(output_file_name) + "_single"), view=show) def to_dict(self): - return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest, - 'name': self.name, 'description': self.description, - 'n_certs': len(self), 'certs': self.certs, 'algs': self.algorithms} + return { + "timestamp": self.timestamp, + "sha256_digest": self.sha256_digest, + "name": self.name, + "description": self.description, + "n_certs": len(self), + "certs": self.certs, + "algs": self.algorithms, + } @classmethod def from_dict(cls, dct: Dict): diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py index 860981f3..e505b217 100644 --- a/sec_certs/dataset/fips_algorithm.py +++ b/sec_certs/dataset/fips_algorithm.py @@ -1,49 +1,50 @@ import json import logging from pathlib import Path -from typing import Dict, Union, List +from typing import Dict, List, Union from bs4 import BeautifulSoup -from sec_certs import helpers as helpers, constants as constants, parallel_processing as cert_processing +from sec_certs import constants as constants +from sec_certs import helpers as helpers +from sec_certs import parallel_processing as cert_processing +from sec_certs.config.configuration import config from sec_certs.dataset.dataset import Dataset -from sec_certs.serialization.json import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder - from sec_certs.sample.fips import FIPSCertificate -from sec_certs.config.configuration import config - +from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder logger = logging.getLogger(__name__) class FIPSAlgorithmDataset(Dataset, ComplexSerializableType): - - certs: Dict[str, List] # type: ignore # noqa + + certs: Dict[str, List] # type: ignore # noqa + def get_certs_from_web(self): self.root_dir.mkdir(exist_ok=True) algs_paths, algs_urls = [], [] # get first page to find out how many pages there are - helpers.download_file( - constants.FIPS_ALG_URL + '1', - self.root_dir / "page1.html") + helpers.download_file(constants.FIPS_ALG_URL + "1", self.root_dir / "page1.html") with open(self.root_dir / "page1.html", "r") as alg_file: - soup = BeautifulSoup(alg_file.read(), 'html.parser') - num_pages = soup.select('span[data-total-pages]')[0].attrs + soup = BeautifulSoup(alg_file.read(), "html.parser") + num_pages = soup.select("span[data-total-pages]")[0].attrs - for i in range(2, int(num_pages['data-total-pages']) + 1): - if not (self.root_dir / f'page{i}.html').exists(): - algs_urls.append( - constants.FIPS_ALG_URL + str(i)) + for i in range(2, int(num_pages["data-total-pages"]) + 1): + if not (self.root_dir / f"page{i}.html").exists(): + algs_urls.append(constants.FIPS_ALG_URL + str(i)) algs_paths.append(self.root_dir / f"page{i}.html") # get the last page, always - helpers.download_file(constants.FIPS_ALG_URL + num_pages['data-total-pages'], - self.root_dir / f"page{int(num_pages['data-total-pages'])}.html") + helpers.download_file( + constants.FIPS_ALG_URL + num_pages["data-total-pages"], + self.root_dir / f"page{int(num_pages['data-total-pages'])}.html", + ) logger.info(f"downloading {len(algs_urls)} algs html files") - cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(algs_urls, algs_paths)), - config.n_threads) + cert_processing.process_parallel( + FIPSCertificate.download_html_page, list(zip(algs_urls, algs_paths)), config.n_threads + ) self.parse_html() @@ -51,78 +52,77 @@ class FIPSAlgorithmDataset(Dataset, ComplexSerializableType): def _extract_algorithm_information(elements, vendor, date, product, validation): for elem in elements: # td > a > (vendor or date) - attachments = elem.find_all('a') + attachments = elem.find_all("a") if len(attachments) == 0: - vendor = elem.text.strip() if 'vendor-name' in elem['id'] else vendor - date = elem.text.strip() if 'validation-date' in elem['id'] else date + vendor = elem.text.strip() if "vendor-name" in elem["id"] else vendor + date = elem.text.strip() if "validation-date" in elem["id"] else date continue for attachment in attachments: - product = elem.text.strip() if 'product-name' in attachment['id'] else product - validation = elem.text.strip() if 'validation-number' in attachment['id'] else validation + product = elem.text.strip() if "product-name" in attachment["id"] else product + validation = elem.text.strip() if "validation-number" in attachment["id"] else validation return vendor, date, product, validation def parse_html(self): def split_alg(alg_string): - cert_type = alg_string.rstrip('0123456789') - cert_id = alg_string[len(cert_type):] + cert_type = alg_string.rstrip("0123456789") + cert_id = alg_string[len(cert_type) :] return cert_type.strip(), cert_id.strip() for f in helpers.search_files(self.root_dir): if not f.endswith("html"): continue - with open(f, 'r', encoding='utf-8') as handle: - html_soup = BeautifulSoup(handle.read(), 'html.parser') + with open(f, "r", encoding="utf-8") as handle: + html_soup = BeautifulSoup(handle.read(), "html.parser") - table = html_soup.find( - 'table', class_='table table-condensed publications-table table-bordered') - tbody_contents = table.find('tbody').find_all('tr') + table = html_soup.find("table", class_="table table-condensed publications-table table-bordered") + tbody_contents = table.find("tbody").find_all("tr") vendor = product = validation = date = "" for tr in tbody_contents: - elements = tr.find_all('td') + elements = tr.find_all("td") vendor, date, product, validation = FIPSAlgorithmDataset._extract_algorithm_information( elements, vendor, date, product, validation ) alg_type, alg_id = split_alg(validation) - fips_alg = FIPSCertificate.Algorithm( - alg_id, vendor, product, alg_type, date) + fips_alg = FIPSCertificate.Algorithm(alg_id, vendor, product, alg_type, date) if alg_id not in self.certs: self.certs[alg_id] = [] self.certs[alg_id].append(fips_alg) + def convert_all_pdfs(self): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") def download_all_pdfs(self): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") @property def serialized_attributes(self) -> List[str]: - return ['certs'] + return ["certs"] @classmethod def from_dict(cls, dct: Dict): - certs = dct['certs'] - - directory = dct['_root_dir'] if '_root_dir' in dct else '' - dset = cls(certs, Path(directory), 'algorithms', 'algorithms used in dataset') + certs = dct["certs"] + + directory = dct["_root_dir"] if "_root_dir" in dct else "" + dset = cls(certs, Path(directory), "algorithms", "algorithms used in dataset") return dset def to_dict(self): - return self.__dict__ + return self.__dict__ def to_json(self, output_path: Union[str, Path] = None): if not output_path: output_path = self.json_path - with Path(output_path).open('w') as handle: + with Path(output_path).open("w") as handle: json.dump(self, handle, indent=4, cls=CustomJSONEncoder) @classmethod def from_json(cls, input_path: Union[str, Path]): input_path = Path(input_path) - with input_path.open('r') as handle: + with input_path.open("r") as handle: dset = json.load(handle, cls=CustomJSONDecoder) dset.root_dir = input_path.parent.absolute() return dset diff --git a/sec_certs/dataset/protection_profile.py b/sec_certs/dataset/protection_profile.py index eae3cb64..4e7ba747 100644 --- a/sec_certs/dataset/protection_profile.py +++ b/sec_certs/dataset/protection_profile.py @@ -1,9 +1,9 @@ -from dataclasses import dataclass -from typing import Dict, Tuple, Union, Optional, ClassVar -from pathlib import Path import json import logging import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar, Dict, Optional, Tuple, Union import sec_certs.helpers as helpers from sec_certs.sample.protection_profile import ProtectionProfile @@ -13,17 +13,17 @@ logger = logging.getLogger(__name__) @dataclass class ProtectionProfileDataset: - static_dataset_url: ClassVar[str] = 'https://ajanovsky.cz/pp_data_complete_processed.json' + static_dataset_url: ClassVar[str] = "https://ajanovsky.cz/pp_data_complete_processed.json" - pps: Dict[Tuple[str, str], ProtectionProfile] + pps: Dict[Tuple[str, Optional[str]], ProtectionProfile] def __iter__(self): yield from self.pps.values() - def __getitem__(self, item: Tuple[str, str]) -> ProtectionProfile: + def __getitem__(self, item: Tuple[str, Optional[str]]) -> ProtectionProfile: return self.pps.__getitem__(item) - def __setitem__(self, key: Tuple[str, str], value: ProtectionProfile): + def __setitem__(self, key: Tuple[str, Optional[str]], value: ProtectionProfile): self.pps.__setitem__(key, value) def __contains__(self, key): @@ -34,24 +34,24 @@ class ProtectionProfileDataset: @classmethod def from_json(cls, json_path: Union[str, Path]): - with Path(json_path).open('r') as handle: + with Path(json_path).open("r") as handle: data = json.load(handle) pps = [ProtectionProfile.from_old_api_dict(x) for x in data.values()] dct = {} for item in pps: if (item.pp_name, item.pp_link) in dct: - logger.warning(f'Duplicate entry in PP dataset: {(item.pp_name, item.pp_link)}') + logger.warning(f"Duplicate entry in PP dataset: {(item.pp_name, item.pp_link)}") dct[(item.pp_name, item.pp_link)] = item return cls(dct) @classmethod - def from_web(cls, store_dataset_path: Optional[Path]): - logger.info(f'Downloading static PP dataset from: {cls.static_dataset_url}') + def from_web(cls, store_dataset_path: Optional[Path] = None): + logger.info(f"Downloading static PP dataset from: {cls.static_dataset_url}") if not store_dataset_path: tmp = tempfile.TemporaryDirectory() - store_dataset_path = Path(tmp.name) + store_dataset_path = Path(tmp.name) / "pp_dataset.json" helpers.download_file(cls.static_dataset_url, store_dataset_path) obj = cls.from_json(store_dataset_path) diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 2816becb..3637fe38 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -1,44 +1,45 @@ -import os -import re -from typing import Sequence, Tuple, Optional, Set, List, Dict, Hashable, Any -import logging -import pikepdf -import requests -from multiprocessing.pool import ThreadPool -from pathlib import Path -from tqdm import tqdm as tqdm_original import hashlib import html -from typing import Union, List -from datetime import date -import numpy as np -import pandas as pd +import logging +import os +import re import subprocess import time -import copy -from packaging.version import VERSION_PATTERN - - - +from datetime import date from enum import Enum +from multiprocessing.pool import ThreadPool +from pathlib import Path +from typing import Dict, Hashable, List, Optional, Sequence, Set, Tuple, Union + import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pikepdf +import requests from PyPDF2 import PdfFileReader +from tqdm import tqdm as tqdm_original -import sec_certs.constants import sec_certs.constants as constants -from sec_certs.config.configuration import config from sec_certs.cert_rules import REGEXEC_SEP from sec_certs.cert_rules import rules as cc_search_rules -from sec_certs.constants import TAG_MATCH_COUNTER, APPEND_DETAILED_MATCH_MATCHES, TAG_MATCH_MATCHES, \ - FILE_ERRORS_STRATEGY, LINE_SEPARATOR +from sec_certs.config.configuration import config +from sec_certs.constants import ( + APPEND_DETAILED_MATCH_MATCHES, + FILE_ERRORS_STRATEGY, + LINE_SEPARATOR, + TAG_MATCH_COUNTER, + TAG_MATCH_MATCHES, +) logger = logging.getLogger(__name__) + def tqdm(*args, **kwargs): if "disable" in kwargs: return tqdm_original(*args, **kwargs) return tqdm_original(*args, **kwargs, disable=not config.enable_progress_bars) + def download_file(url: str, output: Path, delay: float = 0) -> Union[str, int]: try: time.sleep(delay) @@ -50,7 +51,7 @@ def download_file(url: str, output: Path, delay: float = 0) -> Union[str, int]: except requests.exceptions.Timeout: return requests.codes.timeout except Exception as e: - logger.error(f'Failed to download from {url}; {e}') + logger.error(f"Failed to download from {url}; {e}") return constants.RETURNCODE_NOK @@ -71,13 +72,13 @@ def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Se def get_first_16_bytes_sha256(string: str) -> str: - return hashlib.sha256(string.encode('utf-8')).hexdigest()[:16] + return hashlib.sha256(string.encode("utf-8")).hexdigest()[:16] def get_sha256_filepath(filepath): hash_sha256 = hashlib.sha256() with open(filepath, "rb") as f: - for chunk in iter(lambda: f.read(4096), b''): + for chunk in iter(lambda: f.read(4096), b""): hash_sha256.update(chunk) return hash_sha256.hexdigest() @@ -85,7 +86,7 @@ def get_sha256_filepath(filepath): def sanitize_link(record: Optional[str]) -> Optional[str]: if not record: return None - return record.replace(':443', '').replace(' ', '%20').replace('http://', 'https://') + return record.replace(":443", "").replace(" ", "%20").replace("http://", "https://") def sanitize_date(record: Union[pd.Timestamp, date, np.datetime64]) -> Union[date, None]: @@ -97,24 +98,21 @@ def sanitize_date(record: Union[pd.Timestamp, date, np.datetime64]) -> Union[dat return record # type: ignore -def sanitize_string(record: Optional[str]) -> Optional[str]: - if not record: - return None - else: - # TODO: There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I &#38; J' that has to be unescaped twice - string = html.unescape(html.unescape(record)).replace('\n', '') - return ' '.join(string.split()) +def sanitize_string(record: str) -> str: + # TODO: There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I &#38; J' that has to be unescaped twice + string = html.unescape(html.unescape(record)).replace("\n", "") + return " ".join(string.split()) def sanitize_security_levels(record: Union[str, set]) -> set: if isinstance(record, str): - record = set(record.split(',')) + record = set(record.split(",")) - if 'PP\xa0Compliant' in record: - record.remove('PP\xa0Compliant') + if "PP\xa0Compliant" in record: + record.remove("PP\xa0Compliant") - if 'None' in record: - record.remove('None') + if "None" in record: + record.remove("None") return record @@ -122,7 +120,7 @@ def sanitize_security_levels(record: Union[str, set]) -> set: def sanitize_protection_profiles(record: str) -> list: if not record: return [] - return record.split(',') + return record.split(",") # TODO: realize whether this stays or goes somewhere else @@ -135,23 +133,23 @@ def parse_list_of_tables(txt: str) -> Set[str]: rr = re.compile(r"^.+?(?:[Ff]unction|[Aa]lgorithm|[Ss]ecurity [Ff]unctions?).+?(?P<page_num>\d+)$", re.MULTILINE) pages = set() for m in rr.finditer(txt): - pages.add(m.group('page_num')) + pages.add(m.group("page_num")) return pages def find_tables_iterative(file_text: str) -> List[int]: current_page = 1 pages = set() - for line in file_text.split('\n'): - if '\f' in line: + for line in file_text.split("\n"): + if "\f" in line: current_page += 1 - if line.startswith('Table ') or line.startswith('Exhibit'): + if line.startswith("Table ") or line.startswith("Exhibit"): pages.add(current_page) pages.add(current_page + 1) if current_page > 2: pages.add(current_page - 1) if not pages: - logger.warning('No pages found') + logger.warning("No pages found") for page in pages: if page > current_page - 1: return list(pages - {page}) @@ -178,7 +176,7 @@ def find_tables(txt: str, file_name: Path) -> Optional[List]: return None # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer - logger.info(f'parsing tables in {file_name}') + logger.info(f"parsing tables in {file_name}") table_page_indices = find_tables_iterative(txt) return table_page_indices if table_page_indices else None @@ -195,7 +193,9 @@ def repair_pdf(file: Path): def convert_pdf_file(pdf_path: Path, txt_path: Path, options): - response = subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60).returncode + response = subprocess.run( + ["pdftotext", *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60 + ).returncode if response == 0: return constants.RETURNCODE_OK else: @@ -206,18 +206,18 @@ def extract_pdf_metadata(filepath: Path): metadata = dict() try: - metadata['pdf_file_size_bytes'] = filepath.stat().st_size - with filepath.open('rb') as handle: + metadata["pdf_file_size_bytes"] = filepath.stat().st_size + with filepath.open("rb") as handle: pdf = PdfFileReader(handle) - metadata['pdf_is_encrypted'] = pdf.getIsEncrypted() - metadata['pdf_number_of_pages'] = pdf.getNumPages() + metadata["pdf_is_encrypted"] = pdf.getIsEncrypted() + metadata["pdf_number_of_pages"] = pdf.getNumPages() pdf_document_info = pdf.getDocumentInfo() metadata.update({key: str(val) for key, val in pdf_document_info.items()} if pdf_document_info else {}) except Exception as e: - error_msg = f'Failed to read metadata of {filepath}, error: {e}' + error_msg = f"Failed to read metadata of {filepath}, error: {e}" logger.error(error_msg) return error_msg, None @@ -225,7 +225,7 @@ def extract_pdf_metadata(filepath: Path): # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! -def search_only_headers_anssi(filepath: Path): +def search_only_headers_anssi(filepath: Path): # noqa: C901 class HEADER_TYPE(Enum): HEADER_FULL = 1 HEADER_MISSING_CERT_ITEM_VERSION = 2 @@ -233,110 +233,185 @@ def search_only_headers_anssi(filepath: Path): HEADER_DUPLICITIES = 4 rules_certificate_preface = [ - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur\\(s\\)(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d\’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur\\(s\\)(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\\(s\\)(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name \\(reference / version\\)(.+)TOE name \\(reference / version\\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)TOE name(.+)Product\'s reference/ version(.+)TOE\'s reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements'), - + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d\’évaluation(.+)Accords de reconnaissance applicables", # noqa: W605 + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", # noqa: W605 + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\\(s\\)(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name \\(reference / version\\)(.+)TOE name \\(reference / version\\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)TOE name(.+)Product's reference/ version(.+)TOE's reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements", + ), # corrupted text (duplicities) - (HEADER_TYPE.HEADER_DUPLICITIES, - 'Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles'), - + ( + HEADER_TYPE.HEADER_DUPLICITIES, + "Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles", + ), # rules without product version - (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - 'Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - 'Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - 'Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - + ( + HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, + "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, + "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, + "Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), # rules without protection profile - (HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), + ( + HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), ] - # statistics about rules success rate num_rules_hits = {} for rule in rules_certificate_preface: num_rules_hits[rule[1]] = 0 - items_found = {} # type: ignore # noqa + items_found = {} # type: ignore # noqa try: whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath) # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs - pos = whole_text.find(' ') + pos = whole_text.find(" ") if pos != -1: - pos = whole_text.find(' ', pos) + pos = whole_text.find(" ", pos) if pos != -1: whole_text = whole_text[pos:] @@ -359,7 +434,7 @@ def search_only_headers_anssi(filepath: Path): if not other_rule_already_match: other_rule_already_match = True else: - logger.warning(f'WARNING: multiple rules are matching same certification document: {filepath}') + logger.warning(f"WARNING: multiple rules are matching same certification document: {filepath}") num_rules_hits[rule[1]] += 1 # add hit to this rule match_groups = m.groups() @@ -371,15 +446,17 @@ def search_only_headers_anssi(filepath: Path): index_next_item += 1 if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION: - items_found[constants.TAG_CERT_ITEM_VERSION] = '' + items_found[constants.TAG_CERT_ITEM_VERSION] = "" else: items_found[constants.TAG_CERT_ITEM_VERSION] = normalize_match_string(match_groups[index_next_item]) index_next_item += 1 if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES: - items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = '' + items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = "" else: - items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(match_groups[index_next_item]) + items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( + match_groups[index_next_item] + ) index_next_item += 1 items_found[constants.TAG_CC_VERSION] = normalize_match_string(match_groups[index_next_item]) @@ -394,7 +471,7 @@ def search_only_headers_anssi(filepath: Path): items_found[constants.TAG_CERT_LAB] = normalize_match_string(match_groups[index_next_item]) index_next_item += 1 except Exception as e: - error_msg = f'Failed to parse ANSSI frontpage headers from {filepath}; {e}' + error_msg = f"Failed to parse ANSSI frontpage headers from {filepath}; {e}" logger.error(error_msg) return error_msg, None @@ -410,21 +487,24 @@ def search_only_headers_anssi(filepath: Path): return constants.RETURNCODE_OK, items_found + # TODO: Please refactor me. I need it so badlyyyyyy!!! -def search_only_headers_bsi(filepath: Path): - LINE_SEPARATOR_STRICT = ' ' +def search_only_headers_bsi(filepath: Path): # noqa: C901 + LINE_SEPARATOR_STRICT = " " NUM_LINES_TO_INVESTIGATE = 15 rules_certificate_preface = [ - '(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)', - '(BSI-DSZ-CC-.+?) zu (.+?) der (.*)', + "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)", + "(BSI-DSZ-CC-.+?) zu (.+?) der (.*)", ] - items_found = {} # type: ignore # noqa + items_found = {} # type: ignore # noqa no_match_yet = True try: # Process front page with info: cert_id, certified_item and developer - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT) + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file( + filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT + ) for rule in rules_certificate_preface: rule_and_sep = rule + REGEXEC_SEP @@ -443,32 +523,34 @@ def search_only_headers_bsi(filepath: Path): certified_item = match_groups[1] developer = match_groups[2] - FROM_KEYWORD_LIST = [' from ', ' der '] + FROM_KEYWORD_LIST = [" from ", " der "] for from_keyword in FROM_KEYWORD_LIST: from_keyword_len = len(from_keyword) if certified_item.find(from_keyword) != -1: - logger.warning(f'string {from_keyword} detected in certified item - shall not be here, fixing...') - certified_item_first = certified_item[:certified_item.find(from_keyword)] - developer = certified_item[certified_item.find(from_keyword) + from_keyword_len:] + logger.warning( + f"string {from_keyword} detected in certified item - shall not be here, fixing..." + ) + certified_item_first = certified_item[: certified_item.find(from_keyword)] + developer = certified_item[certified_item.find(from_keyword) + from_keyword_len :] certified_item = certified_item_first continue - end_pos = developer.find('\f-') + end_pos = developer.find("\f-") if end_pos == -1: - end_pos = developer.find('\fBSI') + end_pos = developer.find("\fBSI") if end_pos == -1: - end_pos = developer.find('Bundesamt') + end_pos = developer.find("Bundesamt") if end_pos != -1: developer = developer[:end_pos] items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item) items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer) - items_found[constants.TAG_CERT_LAB] = 'BSI' + items_found[constants.TAG_CERT_LAB] = "BSI" # Process page with more detailed sample info # PP Conformance, Functionality, Assurance - rules_certificate_third = ['PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified'] + rules_certificate_third = ["PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified"] whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath) @@ -477,15 +559,17 @@ def search_only_headers_bsi(filepath: Path): for m in re.finditer(rule_and_sep, whole_text): # check if previous rules had at least one match - if not constants.TAG_CERT_ID in items_found.keys(): - logger.error('ERROR: front page not found for file: {}'.format(filepath)) + if constants.TAG_CERT_ID not in items_found.keys(): + logger.error("ERROR: front page not found for file: {}".format(filepath)) match_groups = m.groups() ref_protection_profiles = match_groups[0] cc_version = match_groups[1] cc_security_level = match_groups[2] - items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(ref_protection_profiles) + items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( + ref_protection_profiles + ) items_found[constants.TAG_CC_VERSION] = normalize_match_string(cc_version) items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(cc_security_level) @@ -495,7 +579,7 @@ def search_only_headers_bsi(filepath: Path): # print('Total no hits files: {}'.format(len(files_without_match))) # print('\n**********************************') except Exception as e: - error_msg = f'Failed to parse BSI headers from frontpage: {filepath}; {e}' + error_msg = f"Failed to parse BSI headers from frontpage: {filepath}; {e}" logger.error(error_msg) return error_msg, None @@ -504,7 +588,7 @@ def search_only_headers_bsi(filepath: Path): def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, int]]]]: try: - result = parse_cert_file(filepath, cc_search_rules, -1, sec_certs.constants.LINE_SEPARATOR)[0] + result = parse_cert_file(filepath, cc_search_rules, -1, constants.LINE_SEPARATOR)[0] processed_result = {} top_level_keys = list(result.keys()) @@ -512,13 +596,22 @@ def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, processed_result[key] = {key: val for key, val in gen_dict_extract(result[key])} except Exception as e: - error_msg = f'Failed to parse keywords from: {filepath}; {e}' + error_msg = f"Failed to parse keywords from: {filepath}; {e}" logger.error(error_msg) return error_msg, None return constants.RETURNCODE_OK, processed_result -def plot_dataframe_graph(data: Dict, label: str, file_name: str, density: bool = False, cumulative: bool = False, bins: int = 50, log: bool = True, show: bool = True): +def plot_dataframe_graph( + data: Dict, + label: str, + file_name: str, + density: bool = False, + cumulative: bool = False, + bins: int = 50, + log: bool = True, + show: bool = True, +): pd_data = pd.Series(data) pd_data.hist(bins=bins, label=label, density=density, cumulative=cumulative) plt.savefig(file_name) @@ -554,15 +647,17 @@ def save_modified_cert_file(target_file, modified_cert_file_text, is_unicode_tex try: write_file.write(modified_cert_file_text) - except UnicodeEncodeError as e: - print('UnicodeDecodeError while writing file fragments back') + except UnicodeEncodeError: + print("UnicodeDecodeError while writing file fragments back") finally: write_file.close() -def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR): +# TODO: Please, refactor me. +def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR): # noqa: C901 whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file( - file_name, limit_max_lines, line_separator) + file_name, limit_max_lines, line_separator + ) # apply all rules items_found_all = {} @@ -580,9 +675,9 @@ def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator= rule_str = rule rule_and_sep = rule + REGEXEC_SEP - #matches_with_newlines_count = sum(1 for _ in re.finditer(rule_and_sep, whole_text_with_newlines)) - #matches_without_newlines_count = sum(1 for _ in re.finditer(rule_and_sep, whole_text)) - #for m in re.finditer(rule_and_sep, whole_text_with_newlines): + # matches_with_newlines_count = sum(1 for _ in re.finditer(rule_and_sep, whole_text_with_newlines)) + # matches_without_newlines_count = sum(1 for _ in re.finditer(rule_and_sep, whole_text)) + # for m in re.finditer(rule_and_sep, whole_text_with_newlines): for m in re.finditer(rule_and_sep, whole_text): # insert rule if at least one match for it was found if rule not in items_found: @@ -594,7 +689,7 @@ def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator= MAX_ALLOWED_MATCH_LENGTH = 300 match_len = len(match) if match_len > MAX_ALLOWED_MATCH_LENGTH: - print('WARNING: Excessive match with length of {} detected for rule {}'.format(match_len, rule)) + print("WARNING: Excessive match with length of {} detected for rule {}".format(match_len, rule)) if match not in items_found[rule_str]: items_found[rule_str][match] = {} @@ -611,8 +706,7 @@ def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator= # start index, end index, line number # items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number]) if APPEND_DETAILED_MATCH_MATCHES: - items_found[rule_str][match][TAG_MATCH_MATCHES].append( - [match_span[0], match_span[1]]) + items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1]]) # highlight all found strings (by xxxxx) from the input text and store the rest all_matches = [] @@ -626,31 +720,30 @@ def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator= # sort before replacement based on the length of match all_matches.sort(key=len, reverse=True) for match in all_matches: - whole_text_with_newlines = whole_text_with_newlines.replace( - match, 'x' * len(match)) + whole_text_with_newlines = whole_text_with_newlines.replace(match, "x" * len(match)) return items_found_all, (whole_text_with_newlines, was_unicode_decode_error) def normalize_match_string(match): - match = match.strip().rstrip('];.”":)(,').rstrip(os.sep).replace(' ', ' ') - return ''.join(filter(str.isprintable, match)) + match = match.strip().rstrip('];.”":)(,').rstrip(os.sep).replace(" ", " ") + return "".join(filter(str.isprintable, match)) def load_cert_file(file_name, limit_max_lines=-1, line_separator=LINE_SEPARATOR): lines = [] was_unicode_decode_error = False - with open(file_name, 'r', errors=FILE_ERRORS_STRATEGY) as f: + with open(file_name, "r", errors=FILE_ERRORS_STRATEGY) as f: try: lines = f.readlines() except UnicodeDecodeError: f.close() was_unicode_decode_error = True - print(' WARNING: UnicodeDecodeError, opening as utf8') + print(" WARNING: UnicodeDecodeError, opening as utf8") with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2: # coding failure, try line by line - line = ' ' + line = " " while line: try: line = f2.readline() @@ -659,19 +752,19 @@ def load_cert_file(file_name, limit_max_lines=-1, line_separator=LINE_SEPARATOR) # ignore error continue - whole_text = '' - whole_text_with_newlines = '' + whole_text = "" + whole_text_with_newlines = "" # we will estimate the line for searched matches # => we need to known how much lines were modified (removal of eoln..) # for removed newline and for any added separator - line_length_compensation = 1 - len(LINE_SEPARATOR) + # line_length_compensation = 1 - len(LINE_SEPARATOR) lines_included = 0 for line in lines: if limit_max_lines != -1 and lines_included >= limit_max_lines: break whole_text_with_newlines += line - line = line.replace('\n', '') + line = line.replace("\n", "") whole_text += line whole_text += line_separator lines_included += 1 @@ -680,7 +773,7 @@ def load_cert_file(file_name, limit_max_lines=-1, line_separator=LINE_SEPARATOR) def load_cert_html_file(file_name): - with open(file_name, 'r', errors=FILE_ERRORS_STRATEGY) as f: + with open(file_name, "r", errors=FILE_ERRORS_STRATEGY) as f: try: whole_text = f.read() except UnicodeDecodeError: @@ -689,11 +782,11 @@ def load_cert_html_file(file_name): try: whole_text = f2.read() except UnicodeDecodeError: - print('### ERROR: failed to read file {}'.format(file_name)) + print("### ERROR: failed to read file {}".format(file_name)) return whole_text -def gen_dict_extract(dct: Dict, searched_key: Hashable = 'count'): +def gen_dict_extract(dct: Dict, searched_key: Hashable = "count"): """ Function to flatten dictionary with some serious limitations. We only expect to use it temporarily on dictionary produced by extract_keywords that contains many layers. On the deepest level in that dictionary, 'some_match': {'count': frequency}. @@ -712,18 +805,19 @@ def gen_dict_extract(dct: Dict, searched_key: Hashable = 'count'): else: yield key, result + def compute_heuristics_version(cert_name: str) -> List[str]: """ Will extract possible versions from the name of sample """ - at_least_something = r'(\b(\d)+\b)' - just_numbers = r'(\d{1,5})(\.\d{1,5})' + at_least_something = r"(\b(\d)+\b)" + just_numbers = r"(\d{1,5})(\.\d{1,5})" - without_version = r'(' + just_numbers + r'+)' - long_version = r'(' + r'(\bversion)\s*' + just_numbers + r'+)' - short_version = r'(' + r'\bv\s*' + just_numbers + r'+)' - full_regex_string = r'|'.join([without_version, short_version, long_version]) - normalizer = r'(\d+\.*)+' + without_version = r"(" + just_numbers + r"+)" + long_version = r"(" + r"(\bversion)\s*" + just_numbers + r"+)" + short_version = r"(" + r"\bv\s*" + just_numbers + r"+)" + full_regex_string = r"|".join([without_version, short_version, long_version]) + normalizer = r"(\d+\.*)+" matched_strings = set([max(x, key=len) for x in re.findall(full_regex_string, cert_name, re.IGNORECASE)]) if not matched_strings: @@ -732,18 +826,21 @@ def compute_heuristics_version(cert_name: str) -> List[str]: # return identified_versions if identified_versions else ['-'] if not matched_strings: - return ['-'] + return ["-"] matched = [re.search(normalizer, x) for x in matched_strings] return [x.group() for x in matched if x is not None] -def tokenize_dataset(dset: List[str], keywords: Set[str]) -> np.ndarray: + +def tokenize_dataset(dset: List[str], keywords: Set[str]) -> np.ndarray: return np.array([tokenize(x, keywords) for x in dset]) + def tokenize(string: str, keywords: Set[str]) -> str: - return ' '.join([x for x in string.split() if x.lower() in keywords]) + return " ".join([x for x in string.split() if x.lower() in keywords]) + def filter_shortly_described_cves(x, y): - n_tokens = np.array(list(map(lambda item: len(item.split(' ')), x))) + n_tokens = np.array(list(map(lambda item: len(item.split(" ")), x))) indices = np.where(n_tokens > 5) return np.array(x)[indices], np.array(y)[indices] diff --git a/sec_certs/model/cpe_matching.py b/sec_certs/model/cpe_matching.py index d76d9b4f..610b409a 100644 --- a/sec_certs/model/cpe_matching.py +++ b/sec_certs/model/cpe_matching.py @@ -1,15 +1,14 @@ -from sklearn.base import BaseEstimator -from typing import Dict, Tuple, Set, List, Optional, Union -from sec_certs.sample.cpe import CPE -import sec_certs.helpers as helpers import itertools -import re -from rapidfuzz import process, fuzz -import operator -from pathlib import Path -import json import logging +import re +from typing import Dict, List, Optional, Set, Tuple + from packaging.version import parse +from rapidfuzz import fuzz +from sklearn.base import BaseEstimator + +import sec_certs.helpers as helpers +from sec_certs.sample.cpe import CPE logger = logging.getLogger(__name__) @@ -20,8 +19,11 @@ class CPEClassifier(BaseEstimator): Adheres to sklearn BaseEstimator interface. Fit method is called on list of CPEs and build two look-up dictionaries, see description of attributes. """ + vendor_to_versions_: Dict[str, Set[str]] # Key: CPE vendor, Value: versions of all CPE records of that vendor - vendor_version_to_cpe_: Dict[Tuple[str, str], Set[CPE]] # Key: (CPE vendor, version), Value: All CPEs that are of (vendor, version) + vendor_version_to_cpe_: Dict[ + Tuple[str, str], Set[CPE] + ] # Key: (CPE vendor, version), Value: All CPEs that are of (vendor, version) vendors_: Set[str] def __init__(self, match_threshold: int = 80, n_max_matches: int = 10): @@ -59,7 +61,7 @@ class CPEClassifier(BaseEstimator): self.vendors_ = set(self.vendor_to_versions_.keys()) self.vendor_version_to_cpe_ = dict() - for cpe in helpers.tqdm(sufficiently_long_cpes, desc='Fitting the CPE classifier'): + for cpe in helpers.tqdm(sufficiently_long_cpes, desc="Fitting the CPE classifier"): self.vendor_to_versions_[cpe.vendor].add(cpe.version) if (cpe.vendor, cpe.version) not in self.vendor_version_to_cpe_: self.vendor_version_to_cpe_[(cpe.vendor, cpe.version)] = {cpe} @@ -72,15 +74,16 @@ class CPEClassifier(BaseEstimator): @param X: tuples (vendor, product name, identified versions in product name) @return: List of CPE uris that correspond to given input, None if nothing was found. """ - return [self.predict_single_cert(x[0], x[1], x[2]) for x in helpers.tqdm(X, desc='Predicting')] + return [self.predict_single_cert(x[0], x[1], x[2]) for x in helpers.tqdm(X, desc="Predicting")] - def predict_single_cert(self, - vendor: str, - product_name: str, - versions: Optional[List[str]], - relax_version: bool = False, - relax_title: bool = False - ) -> Optional[List[str]]: + def predict_single_cert( + self, + vendor: str, + product_name: str, + versions: Optional[List[str]], + relax_version: bool = False, + relax_title: bool = False, + ) -> Optional[List[str]]: """ Predict List of CPE uris for triplet (vendor, product_name, list_of_version). The prediction is made as follows: 1. Sanitize all strings @@ -104,17 +107,30 @@ class CPEClassifier(BaseEstimator): ratings = [self.compute_best_match(cpe, sanitized_product_name, candidate_vendors, versions, relax_title=relax_title) for cpe in candidates] # type: ignore threshold = self.match_threshold if not relax_version else 100 final_matches_aux: List[Tuple[float, CPE]] = list(filter(lambda x: x[0] >= threshold, zip(ratings, candidates))) - final_matches: Optional[List[str]] = [x[1].uri for x in final_matches_aux[:self.n_max_matches] if x[1].uri is not None] + final_matches: Optional[List[str]] = [ + x[1].uri for x in final_matches_aux[: self.n_max_matches] if x[1].uri is not None + ] if not relax_title and not final_matches: - final_matches = self.predict_single_cert(vendor, product_name, versions, relax_version=relax_version, relax_title=True) + final_matches = self.predict_single_cert( + vendor, product_name, versions, relax_version=relax_version, relax_title=True + ) if not relax_version and not final_matches: - final_matches = self.predict_single_cert(vendor, product_name, ['-'], relax_version=True, relax_title=relax_title) + final_matches = self.predict_single_cert( + vendor, product_name, ["-"], relax_version=True, relax_title=relax_title + ) return final_matches if final_matches else None - def compute_best_match(self, cpe: CPE, product_name: str, candidate_vendors: List[str], versions: Optional[List[str]], relax_title: bool = False) -> float: + def compute_best_match( + self, + cpe: CPE, + product_name: str, + candidate_vendors: List[str], + versions: Optional[List[str]], + relax_title: bool = False, + ) -> float: """ Tries several different settings in which string similarity between CPE and certificate name is tested. For definition of string similarity, see rapidfuzz package on GitHub. Both token set ratio and partial ratio are tested, @@ -126,7 +142,7 @@ class CPEClassifier(BaseEstimator): @return: Maximal value of the four string similarities discussed above. """ if relax_title: - sanitized_title = CPEClassifier._fully_sanitize_string(cpe.title) if cpe.title else CPEClassifier._fully_sanitize_string(cpe.vendor + ' ' + cpe.item_name + ' ' + cpe.version + ' ' + cpe.update + ' ' + cpe.target_hw) # type: ignore + sanitized_title = CPEClassifier._fully_sanitize_string(cpe.title) if cpe.title else CPEClassifier._fully_sanitize_string(cpe.vendor + " " + cpe.item_name + " " + cpe.version + " " + cpe.update + " " + cpe.target_hw) # type: ignore else: if cpe.title: sanitized_title = CPEClassifier._fully_sanitize_string(cpe.title) @@ -152,16 +168,16 @@ class CPEClassifier(BaseEstimator): @staticmethod def _replace_special_chars_with_space(string: str) -> str: - return re.sub(r'[^a-zA-Z0-9 \n\.]', ' ', string) + return re.sub(r"[^a-zA-Z0-9 \n\.]", " ", string) @staticmethod def _discard_trademark_symbols(string: str) -> str: - return string.replace('®', '').replace('™', '') + return string.replace("®", "").replace("™", "") @staticmethod def _strip_manufacturer_and_version(string: str, manufacturers: List[str], versions: List[str]) -> str: for x in manufacturers + versions: - string = string.lower().replace(CPEClassifier._replace_special_chars_with_space(x.lower()), '').strip() + string = string.lower().replace(CPEClassifier._replace_special_chars_with_space(x.lower()), "").strip() return string def get_candidate_list_of_vendors(self, manufacturer: str) -> Optional[List[str]]: @@ -174,10 +190,12 @@ class CPEClassifier(BaseEstimator): return None result: Set = set() - splits = re.compile(r'[,/]').findall(manufacturer) + splits = re.compile(r"[,/]").findall(manufacturer) if splits: - vendor_tokens = set(itertools.chain.from_iterable([[x.strip() for x in manufacturer.split(s)] for s in splits])) + vendor_tokens = set( + itertools.chain.from_iterable([[x.strip() for x in manufacturer.split(s)] for s in splits]) + ) result_aux = [self.get_candidate_list_of_vendors(x) for x in vendor_tokens] result_used = list(set(itertools.chain.from_iterable([x for x in result_aux if x]))) return result_used if result_used else None @@ -192,22 +210,24 @@ class CPEClassifier(BaseEstimator): result.add(tokenized[0] + tokenized[1]) # Below are completely manual fixes - if 'hewlett' in tokenized or 'hewlett-packard' in tokenized or manufacturer == 'hewlett packard': - result.add('hp') - if 'thales' in tokenized: - result.add('thalesesecurity') - result.add('thalesgroup') - if 'stmicroelectronics' in tokenized: - result.add('st') - if 'athena' in tokenized and 'smartcard' in tokenized: - result.add('athena-scs') - if tokenized[0] == 'the' and not result: - candidate_result = self.get_candidate_list_of_vendors(' '.join(tokenized[1:])) + if "hewlett" in tokenized or "hewlett-packard" in tokenized or manufacturer == "hewlett packard": + result.add("hp") + if "thales" in tokenized: + result.add("thalesesecurity") + result.add("thalesgroup") + if "stmicroelectronics" in tokenized: + result.add("st") + if "athena" in tokenized and "smartcard" in tokenized: + result.add("athena-scs") + if tokenized[0] == "the" and not result: + candidate_result = self.get_candidate_list_of_vendors(" ".join(tokenized[1:])) return list(candidate_result) if candidate_result else None return list(result) if result else None - def get_candidate_vendor_version_pairs(self, cert_candidate_cpe_vendors: List[str], cert_candidate_versions: List[str]) -> Optional[List[Tuple[str, str]]]: + def get_candidate_vendor_version_pairs( + self, cert_candidate_cpe_vendors: List[str], cert_candidate_versions: List[str] + ) -> Optional[List[Tuple[str, str]]]: """ Given parameters, will return Pairs (cpe_vendor, cpe_version) that are relevant to a given sample @param cert_candidate_cpe_vendors: list of CPE vendors relevant to a sample @@ -220,13 +240,17 @@ class CPEClassifier(BaseEstimator): if seeked_version == checked_string: return True else: - return checked_string.startswith(seeked_version) and not checked_string[len(seeked_version)].isdigit() + return ( + checked_string.startswith(seeked_version) and not checked_string[len(seeked_version)].isdigit() + ) if not cpe_version: return False - just_numbers = r'(\d{1,5})(\.\d{1,5})' # TODO: The use of this should be double-checked + just_numbers = r"(\d{1,5})(\.\d{1,5})" # TODO: The use of this should be double-checked for v in cert_versions: - if (simple_startswith(v, cpe_version) and re.search(just_numbers, cpe_version)) or simple_startswith(cpe_version, v): + if (simple_startswith(v, cpe_version) and re.search(just_numbers, cpe_version)) or simple_startswith( + cpe_version, v + ): return True return False @@ -236,7 +260,9 @@ class CPEClassifier(BaseEstimator): candidate_vendor_version_pairs: List[Tuple[str, str]] = [] for vendor in cert_candidate_cpe_vendors: viable_cpe_versions = self.vendor_to_versions_.get(vendor, set()) - matched_cpe_versions = [x for x in viable_cpe_versions if is_cpe_version_among_cert_versions(x, cert_candidate_versions)] + matched_cpe_versions = [ + x for x in viable_cpe_versions if is_cpe_version_among_cert_versions(x, cert_candidate_versions) + ] candidate_vendor_version_pairs.extend([(vendor, x) for x in matched_cpe_versions]) return candidate_vendor_version_pairs @@ -260,4 +286,10 @@ class CPEClassifier(BaseEstimator): @return: """ candidate_vendor_version_pairs = self.get_candidate_vendor_version_pairs(candidate_vendors, candidate_versions) - return list(itertools.chain.from_iterable([self.vendor_version_to_cpe_[x] for x in candidate_vendor_version_pairs])) if candidate_vendor_version_pairs else [] + return ( + list( + itertools.chain.from_iterable([self.vendor_version_to_cpe_[x] for x in candidate_vendor_version_pairs]) + ) + if candidate_vendor_version_pairs + else [] + ) diff --git a/sec_certs/model/dependency_finder.py b/sec_certs/model/dependency_finder.py index 75d6ac4a..07ccb199 100644 --- a/sec_certs/model/dependency_finder.py +++ b/sec_certs/model/dependency_finder.py @@ -1,4 +1,5 @@ -from typing import List, Set, Dict, Tuple, Union, Optional +from typing import Dict, List, Optional, Set, Tuple, Union + from sec_certs.sample.common_criteria import CommonCriteriaCert Certificates = Dict[str, CommonCriteriaCert] @@ -8,7 +9,6 @@ Dependencies = Dict[str, Dict[str, Union[Optional[List[str]], Optional[Set[str]] class DependencyFinder: - def __init__(self): self.dependencies: Dependencies = {} @@ -53,8 +53,9 @@ class DependencyFinder: for referencing in tmp_referenced_by_indirect_nums: if referencing in referenced_by.keys(): tmp_referencing = referenced_by_indirect[referencing].copy() - newly_discovered_references = [x for x in tmp_referencing if - x not in referenced_by_indirect[cert_id]] + newly_discovered_references = [ + x for x in tmp_referencing if x not in referenced_by_indirect[cert_id] + ] referenced_by_indirect[cert_id].update(newly_discovered_references) new_change_detected = True if newly_discovered_references else False @@ -98,17 +99,21 @@ class DependencyFinder: if not cert_id: continue - self.dependencies[dgst]["directly_affected_by"] = \ - DependencyFinder._get_affected_directly(cert_id, referenced_by_direct) + self.dependencies[dgst]["directly_affected_by"] = DependencyFinder._get_affected_directly( + cert_id, referenced_by_direct + ) - self.dependencies[dgst]["indirectly_affected_by"] = \ - DependencyFinder._get_affected_indirectly(cert_id, referenced_by_indirect) + self.dependencies[dgst]["indirectly_affected_by"] = DependencyFinder._get_affected_indirectly( + cert_id, referenced_by_indirect + ) - self.dependencies[dgst]["directly_affecting"] = \ - DependencyFinder._get_affecting_directly(cert_id, referenced_by_direct) + self.dependencies[dgst]["directly_affecting"] = DependencyFinder._get_affecting_directly( + cert_id, referenced_by_direct + ) - self.dependencies[dgst]["indirectly_affecting"] = \ - DependencyFinder._get_affecting_indirectly(cert_id, referenced_by_indirect) + self.dependencies[dgst]["indirectly_affecting"] = DependencyFinder._get_affecting_indirectly( + cert_id, referenced_by_indirect + ) def get_directly_affected_by(self, dgst: str) -> Optional[List[str]]: res = self.dependencies[dgst].get("directly_affected_by", None) diff --git a/sec_certs/model/evaluation.py b/sec_certs/model/evaluation.py index f0c98975..0ef23dd9 100644 --- a/sec_certs/model/evaluation.py +++ b/sec_certs/model/evaluation.py @@ -1,21 +1,21 @@ import json -from pathlib import Path import logging -from typing import List, Set, Union, Optional +from pathlib import Path +from typing import List, Optional, Set, Union + +import numpy as np +import sec_certs.helpers as helpers +from sec_certs.dataset.cpe import CPEDataset from sec_certs.sample.common_criteria import CommonCriteriaCert from sec_certs.sample.fips import FIPSCertificate -from sec_certs.dataset.cpe import CPEDataset from sec_certs.serialization.json import CustomJSONEncoder -import sec_certs.helpers as helpers - -import numpy as np logger = logging.getLogger(__name__) def get_validation_dgsts(filepath: Union[str, Path]) -> Set[str]: - with Path(filepath).open('r') as handle: + with Path(filepath).open("r") as handle: return set(json.load(handle)) @@ -33,7 +33,12 @@ def compute_precision(y: np.ndarray, y_pred: np.ndarray, **kwargs): return np.mean(prec) -def evaluate(x_valid: List[Union[CommonCriteriaCert, FIPSCertificate]], y_valid: List[Optional[List[str]]], outpath: Optional[Union[Path, str]], cpe_dset: CPEDataset): +def evaluate( + x_valid: List[Union[CommonCriteriaCert, FIPSCertificate]], + y_valid: List[Optional[List[str]]], + outpath: Optional[Union[Path, str]], + cpe_dset: CPEDataset, +): y_pred = [x.heuristics.cpe_matches for x in x_valid] precision = compute_precision(np.array(y_valid), np.array(y_pred)) @@ -48,15 +53,15 @@ def evaluate(x_valid: List[Union[CommonCriteriaCert, FIPSCertificate]], y_valid: predicted_cpes_set = set(predicted_cpes) if predicted_cpes else set() predicted_cpes_dict = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in predicted_cpes_set} - cert_name = cert.name if isinstance(cert, CommonCriteriaCert) else cert.web_scan.module_name vendor = cert.manufacturer if isinstance(cert, CommonCriteriaCert) else cert.web_scan.vendor - record = {'certificate_name': cert_name, - 'vendor': vendor, - 'heuristic version': helpers.compute_heuristics_version(cert_name) if cert_name else None, - 'predicted_cpes': predicted_cpes_dict, - 'manually_assigned_cpes': verified_cpes_dict - } + record = { + "certificate_name": cert_name, + "vendor": vendor, + "heuristic version": helpers.compute_heuristics_version(cert_name) if cert_name else None, + "predicted_cpes": predicted_cpes_dict, + "manually_assigned_cpes": verified_cpes_dict, + } if verified_cpes_set.issubset(predicted_cpes_set): correctly_classified.append(record) @@ -67,11 +72,17 @@ def evaluate(x_valid: List[Union[CommonCriteriaCert, FIPSCertificate]], y_valid: n_new_certs_with_match += 1 n_newly_identified += len(predicted_cpes_set - verified_cpes_set) - results = {'Precision': precision, 'n_new_certs_with_match': n_new_certs_with_match, - 'n_newly_identified': n_newly_identified, 'correctly_classified': correctly_classified, - 'badly_classified': badly_classified} - logger.info(f'While keeping precision: {precision}, the classifier identified {n_newly_identified} new CPE matches (Found match for {n_new_certs_with_match} certificates that were previously unmatched) compared to baseline.') + results = { + "Precision": precision, + "n_new_certs_with_match": n_new_certs_with_match, + "n_newly_identified": n_newly_identified, + "correctly_classified": correctly_classified, + "badly_classified": badly_classified, + } + logger.info( + f"While keeping precision: {precision}, the classifier identified {n_newly_identified} new CPE matches (Found match for {n_new_certs_with_match} certificates that were previously unmatched) compared to baseline." + ) if outpath: - with Path(outpath).open('w') as handle: - json.dump(results, handle, indent=4, cls=CustomJSONEncoder)
\ No newline at end of file + with Path(outpath).open("w") as handle: + json.dump(results, handle, indent=4, cls=CustomJSONEncoder) diff --git a/sec_certs/parallel_processing.py b/sec_certs/parallel_processing.py index c42c9c16..7a150070 100644 --- a/sec_certs/parallel_processing.py +++ b/sec_certs/parallel_processing.py @@ -1,16 +1,29 @@ -from sec_certs.helpers import tqdm +import time from multiprocessing.pool import ThreadPool -from billiard.pool import Pool from typing import Callable, Iterable, Optional, Union -import time + +from billiard.pool import Pool + +from sec_certs.helpers import tqdm -def process_parallel(func: Callable, items: Iterable, max_workers: int, callback: Optional[Callable] = None, - use_threading: bool = True, progress_bar: bool = True, unpack: bool = False, - progress_bar_desc: Optional[str] = None): +def process_parallel( + func: Callable, + items: Iterable, + max_workers: int, + callback: Optional[Callable] = None, + use_threading: bool = True, + progress_bar: bool = True, + unpack: bool = False, + progress_bar_desc: Optional[str] = None, +): pool: Union[Pool, ThreadPool] = ThreadPool(max_workers) if use_threading else Pool(max_workers) - results = [pool.apply_async(func, (*i,), callback=callback) for i in items] if unpack else [pool.apply_async(func, (i, ), callback=callback) for i in items] + results = ( + [pool.apply_async(func, (*i,), callback=callback) for i in items] + if unpack + else [pool.apply_async(func, (i,), callback=callback) for i in items] + ) if progress_bar is True and items: bar = tqdm(total=len(results), desc=progress_bar_desc) diff --git a/sec_certs/sample/cc_maintenance_update.py b/sec_certs/sample/cc_maintenance_update.py index 01bb11fe..fa4012e3 100644 --- a/sec_certs/sample/cc_maintenance_update.py +++ b/sec_certs/sample/cc_maintenance_update.py @@ -1,6 +1,6 @@ import logging -from typing import Optional, Dict, List, ClassVar, Tuple from datetime import date +from typing import ClassVar, Dict, List, Optional, Tuple import sec_certs.helpers as helpers from sec_certs.sample.common_criteria import CommonCriteriaCert @@ -10,45 +10,86 @@ logger = logging.getLogger(__name__) class CommonCriteriaMaintenanceUpdate(CommonCriteriaCert, ComplexSerializableType): - pandas_columns: ClassVar[List[str]] = ['dgst', 'name', 'report_link', 'st_link', - 'related_cert_digest', 'maintenance_date'] + pandas_columns: ClassVar[List[str]] = [ + "dgst", + "name", + "report_link", + "st_link", + "related_cert_digest", + "maintenance_date", + ] - def __init__(self, name: str, report_link: str, st_link: str, - state: Optional[CommonCriteriaCert.InternalState], - pdf_data: Optional[CommonCriteriaCert.PdfData], - heuristics: Optional[CommonCriteriaCert.CCHeuristics], - related_cert_digest: str, - maintenance_date: date): - super().__init__('', '', name, '', '', '', None, None, - report_link, st_link, '', '', set(), set(), - state, pdf_data, heuristics) + def __init__( + self, + name: str, + report_link: str, + st_link: str, + state: Optional[CommonCriteriaCert.InternalState], + pdf_data: Optional[CommonCriteriaCert.PdfData], + heuristics: Optional[CommonCriteriaCert.CCHeuristics], + related_cert_digest: str, + maintenance_date: date, + ): + super().__init__( + "", + "", + name, + "", + "", + "", + None, + None, + report_link, + st_link, + "", + "", + set(), + set(), + state, + pdf_data, + heuristics, + ) self.related_cert_digest = related_cert_digest self.maintenance_date = maintenance_date @property def serialized_attributes(self) -> List[str]: - return ['dgst'] + list(self.__class__.__init__.__code__.co_varnames)[1:] + return ["dgst"] + list(self.__class__.__init__.__code__.co_varnames)[1:] @property def dgst(self): - return 'cert_' + self.related_cert_digest + '_update_' + helpers.get_first_16_bytes_sha256(self.name) + return "cert_" + self.related_cert_digest + "_update_" + helpers.get_first_16_bytes_sha256(self.name) @property def pandas_tuple(self) -> Tuple: return tuple([getattr(self, x) for x in CommonCriteriaMaintenanceUpdate.pandas_columns]) @classmethod - def from_dict(cls, dct: Dict) -> 'CommonCriteriaMaintenanceUpdate': - dct.pop('dgst') + def from_dict(cls, dct: Dict) -> "CommonCriteriaMaintenanceUpdate": + dct.pop("dgst") return cls(*(tuple(dct.values()))) # TODO: Deserves some inheritance @classmethod def get_updates_from_cc_cert(cls, cert: CommonCriteriaCert): if cert.maintenance_updates is None: raise RuntimeError("Dataset was probably not built correctly - this should not be happening.") - - return [cls(x.maintenance_title, x.maintenance_report_link, x.maintenance_st_link, - None, None, None, cert.dgst, x.maintenance_date) for x in cert.maintenance_updates - if (x.maintenance_title is not None and x.maintenance_report_link is not None\ - and x.maintenance_st_link is not None and x.maintenance_date is not None)] + return [ + cls( + x.maintenance_title, + x.maintenance_report_link, + x.maintenance_st_link, + None, + None, + None, + cert.dgst, + x.maintenance_date, + ) + for x in cert.maintenance_updates + if ( + x.maintenance_title is not None + and x.maintenance_report_link is not None + and x.maintenance_st_link is not None + and x.maintenance_date is not None + ) + ] diff --git a/sec_certs/sample/certificate.py b/sec_certs/sample/certificate.py index b0522d67..b020bffb 100644 --- a/sec_certs/sample/certificate.py +++ b/sec_certs/sample/certificate.py @@ -1,21 +1,20 @@ -import logging -from pathlib import Path import copy -import json import itertools - +import json +import logging from abc import ABC, abstractmethod -from typing import Optional, Union, TypeVar, Type, Any +from pathlib import Path +from typing import Any, Optional, Type, TypeVar, Union -from sec_certs.serialization.json import CustomJSONDecoder, CustomJSONEncoder, ComplexSerializableType -from sec_certs.model.cpe_matching import CPEClassifier from sec_certs.dataset.cve import CVEDataset +from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder logger = logging.getLogger(__name__) class Certificate(ABC, ComplexSerializableType): - T = TypeVar('T', bound='Certificate') + T = TypeVar("T", bound="Certificate") heuristics: Any @@ -26,17 +25,17 @@ class Certificate(ABC, ComplexSerializableType): return str(self.to_dict()) def __str__(self) -> str: - return 'Not implemented' + return "Not implemented" @property @abstractmethod def dgst(self): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") @property @abstractmethod def label_studio_title(self): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") def __eq__(self, other: object) -> bool: if not isinstance(other, Certificate): @@ -44,41 +43,48 @@ class Certificate(ABC, ComplexSerializableType): return self.dgst == other.dgst def to_dict(self): - return {**{'dgst': self.dgst}, **{key: val for key, val in copy.deepcopy(self.__dict__).items() if key in self.serialized_attributes}} + return { + **{"dgst": self.dgst}, + **{key: val for key, val in copy.deepcopy(self.__dict__).items() if key in self.serialized_attributes}, + } @classmethod def from_dict(cls: Type[T], dct: dict) -> T: - dct.pop('dgst') + dct.pop("dgst") return cls(*(tuple(dct.values()))) def to_json(self, output_path: Optional[Union[str, Path]] = None): if output_path is None: - raise RuntimeError(f"You tried to serialize an object ({type(self)}) that does not have implicit json path. Please provide json_path.") - with Path(output_path).open('w') as handle: + raise RuntimeError( + f"You tried to serialize an object ({type(self)}) that does not have implicit json path. Please provide json_path." + ) + with Path(output_path).open("w") as handle: json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False) @classmethod def from_json(cls, input_path: Union[Path, str]): - with Path(input_path).open('r') as handle: + with Path(input_path).open("r") as handle: return json.load(handle, cls=CustomJSONDecoder) @abstractmethod def compute_heuristics_version(self): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") @abstractmethod def compute_heuristics_cpe_vendors(self, cpe_classifier: CPEClassifier): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") @abstractmethod def compute_heuristics_cpe_match(self, cpe_classifier: CPEClassifier): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") def compute_heuristics_related_cves(self, cve_dataset: CVEDataset): if self.heuristics.cpe_matches: related_cves = [cve_dataset.get_cve_ids_for_cpe_uri(x) for x in self.heuristics.cpe_matches] related_cves = list(filter(lambda x: x is not None, related_cves)) if related_cves: - self.heuristics.related_cves = set(itertools.chain.from_iterable([x for x in related_cves if x is not None])) + self.heuristics.related_cves = set( + itertools.chain.from_iterable([x for x in related_cves if x is not None]) + ) else: self.heuristics.related_cves = None diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py index 890c38e8..a353ff61 100644 --- a/sec_certs/sample/common_criteria.py +++ b/sec_certs/sample/common_criteria.py @@ -3,42 +3,40 @@ import operator from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path -from typing import Optional, List, Dict, Tuple, Union, Any, Set, ClassVar - +from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Union import requests from bs4 import Tag -from sec_certs import helpers, constants as constants +from sec_certs import constants as constants +from sec_certs import helpers +from sec_certs.model.cpe_matching import CPEClassifier from sec_certs.sample.certificate import Certificate, logger +from sec_certs.sample.protection_profile import ProtectionProfile from sec_certs.serialization.json import ComplexSerializableType from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.model.cpe_matching import CPEClassifier class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializableType): - cc_url = 'http://www.commoncriteriaportal.org' - empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/' + cc_url = "http://www.commoncriteriaportal.org" + empty_st_url = "http://www.commoncriteriaportal.org/files/epfiles/" @dataclass(eq=True, frozen=True) class MaintenanceReport(ComplexSerializableType): """ Object for holding maintenance reports. """ + maintenance_date: Optional[date] maintenance_title: Optional[str] maintenance_report_link: Optional[str] maintenance_st_link: Optional[str] def __post_init__(self): - super().__setattr__('maintenance_report_link', - helpers.sanitize_link(self.maintenance_report_link)) - super().__setattr__('maintenance_st_link', - helpers.sanitize_link(self.maintenance_st_link)) - super().__setattr__('maintenance_title', - helpers.sanitize_string(self.maintenance_title)) - super().__setattr__('maintenance_date', helpers.sanitize_date(self.maintenance_date)) + super().__setattr__("maintenance_report_link", helpers.sanitize_link(self.maintenance_report_link)) + super().__setattr__("maintenance_st_link", helpers.sanitize_link(self.maintenance_st_link)) + super().__setattr__("maintenance_title", helpers.sanitize_string(self.maintenance_title)) + super().__setattr__("maintenance_date", helpers.sanitize_date(self.maintenance_date)) def __lt__(self, other): return self.maintenance_date < other.maintenance_date @@ -58,10 +56,16 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl st_txt_path: Path report_txt_path: Path - def __init__(self, st_download_ok: bool = True, report_download_ok: bool = True, - st_convert_ok: bool = True, report_convert_ok: bool = True, - st_extract_ok: bool = True, report_extract_ok: bool = True, - errors: Optional[List[str]] = None): + def __init__( + self, + st_download_ok: bool = True, + report_download_ok: bool = True, + st_convert_ok: bool = True, + report_convert_ok: bool = True, + st_extract_ok: bool = True, + report_extract_ok: bool = True, + errors: Optional[List[str]] = None, + ): self.st_download_ok = st_download_ok self.report_download_ok = report_download_ok self.st_convert_ok = st_convert_ok @@ -76,8 +80,15 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl @property def serialized_attributes(self) -> List[str]: - return ['st_download_ok', 'report_download_ok', 'st_convert_ok', 'report_convert_ok', 'st_extract_ok', - 'report_extract_ok', 'errors'] + return [ + "st_download_ok", + "report_download_ok", + "st_convert_ok", + "report_convert_ok", + "st_extract_ok", + "report_extract_ok", + "errors", + ] def report_is_ok_to_download(self, fresh: bool = True): return True if fresh else not self.report_download_ok @@ -117,35 +128,35 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl @property def bsi_data(self) -> Optional[Dict[str, Any]]: - return self.report_frontpage.get('bsi', None) if self.report_frontpage else None + return self.report_frontpage.get("bsi", None) if self.report_frontpage else None @property def anssi_data(self) -> Optional[Dict[str, Any]]: - return self.report_frontpage.get('anssi', None) if self.report_frontpage else None + return self.report_frontpage.get("anssi", None) if self.report_frontpage else None @property def cert_lab(self) -> Optional[List[str]]: labs = [] if bsi_data := self.bsi_data: - labs.append(bsi_data['cert_lab'].split(' ')[0].upper()) + labs.append(bsi_data["cert_lab"].split(" ")[0].upper()) if anssi_data := self.anssi_data: - labs.append(anssi_data['cert_lab'].split(' ')[0].upper()) + labs.append(anssi_data["cert_lab"].split(" ")[0].upper()) return labs if labs else None @property def bsi_cert_id(self) -> Optional[str]: - return self.bsi_data.get('cert_id', None) if self.bsi_data else None + return self.bsi_data.get("cert_id", None) if self.bsi_data else None @property def anssi_cert_id(self) -> Optional[str]: - return self.anssi_data.get('cert_id', None) if self.anssi_data else None + return self.anssi_data.get("cert_id", None) if self.anssi_data else None @property def processed_cert_id(self) -> Optional[str]: if self.bsi_cert_id and self.anssi_cert_id: - logger.error('Both BSI and ANSSI cert_id set.') - raise ValueError('Both BSI and ANSSI cert_id set.') + logger.error("Both BSI and ANSSI cert_id set.") + raise ValueError("Both BSI and ANSSI cert_id set.") if self.bsi_cert_id: return self.bsi_cert_id else: @@ -153,7 +164,7 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl @property def keywords_rules_cert_id(self) -> Optional[Dict[str, Optional[Dict[str, Dict[str, int]]]]]: - return self.report_keywords.get('rules_cert_id', None) if self.report_keywords else None + return self.report_keywords.get("rules_cert_id", None) if self.report_keywords else None @property def keywords_cert_id(self) -> Optional[str]: @@ -164,7 +175,7 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return None candidates = [(x, y) for x, y in self.keywords_rules_cert_id.items()] - candidates = sorted(candidates, key=operator.itemgetter(1), reverse=True) + candidates = sorted(candidates, key=operator.itemgetter(1), reverse=True) return candidates[0][0] @property @@ -191,33 +202,65 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl @property def serialized_attributes(self) -> List[str]: all_vars = copy.deepcopy(super().serialized_attributes) - all_vars.remove('cpe_candidate_vendors') + all_vars.remove("cpe_candidate_vendors") return all_vars def __post_init__(self): self.cpe_candidate_vendors = None - pandas_columns: ClassVar[List[str]] = ['dgst', 'name', 'status', 'category', 'manufacturer', 'scheme', - 'security_level', 'not_valid_before', 'not_valid_after', 'report_link', - 'st_link', 'manufacturer_web', 'extracted_versions', 'cpe_matches', - 'verified_cpe_matches', 'related_cves', 'directly_affected_by', - 'indirectly_affected_by', 'directly_affecting', 'indirectly_affecting'] + pandas_columns: ClassVar[List[str]] = [ + "dgst", + "name", + "status", + "category", + "manufacturer", + "scheme", + "security_level", + "not_valid_before", + "not_valid_after", + "report_link", + "st_link", + "manufacturer_web", + "extracted_versions", + "cpe_matches", + "verified_cpe_matches", + "related_cves", + "directly_affected_by", + "indirectly_affected_by", + "directly_affecting", + "indirectly_affecting", + ] - def __init__(self, status: str, category: str, name: str, manufacturer: Optional[str], scheme: str, - security_level: Union[str, set], not_valid_before: Optional[date], - not_valid_after: Optional[date], report_link: str, st_link: str, cert_link: Optional[str], - manufacturer_web: Optional[str], - protection_profiles: Optional[Set[ProtectionProfile]], - maintenance_updates: Optional[Set[MaintenanceReport]], - state: Optional[InternalState], - pdf_data: Optional[PdfData], - heuristics: Optional[CCHeuristics]): + def __init__( + self, + status: str, + category: str, + name: str, + manufacturer: Optional[str], + scheme: str, + security_level: Union[str, set], + not_valid_before: Optional[date], + not_valid_after: Optional[date], + report_link: str, + st_link: str, + cert_link: Optional[str], + manufacturer_web: Optional[str], + protection_profiles: Optional[Set[ProtectionProfile]], + maintenance_updates: Optional[Set[MaintenanceReport]], + state: Optional[InternalState], + pdf_data: Optional[PdfData], + heuristics: Optional[CCHeuristics], + ): super().__init__() self.status = status self.category = category self.name = helpers.sanitize_string(name) - self.manufacturer = helpers.sanitize_string(manufacturer) + + self.manufacturer = None + if manufacturer: + self.manufacturer = helpers.sanitize_string(manufacturer) + self.scheme = scheme self.security_level = helpers.sanitize_security_levels(security_level) self.not_valid_before = helpers.sanitize_date(not_valid_before) @@ -256,17 +299,34 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl @property def pandas_tuple(self): - return self.dgst, self.name, self.status, self.category, self.manufacturer, self.scheme, self.security_level, \ - self.not_valid_before, self.not_valid_after, self.report_link, self.st_link, self.manufacturer_web, \ - self.heuristics.extracted_versions, self.heuristics.cpe_matches, self.heuristics.verified_cpe_matches, \ - self.heuristics.related_cves, self.heuristics.directly_affected_by, self.heuristics.indirectly_affected_by, \ - self.heuristics.directly_affecting, self.heuristics.indirectly_affecting + return ( + self.dgst, + self.name, + self.status, + self.category, + self.manufacturer, + self.scheme, + self.security_level, + self.not_valid_before, + self.not_valid_after, + self.report_link, + self.st_link, + self.manufacturer_web, + self.heuristics.extracted_versions, + self.heuristics.cpe_matches, + self.heuristics.verified_cpe_matches, + self.heuristics.related_cves, + self.heuristics.directly_affected_by, + self.heuristics.indirectly_affected_by, + self.heuristics.directly_affecting, + self.heuristics.indirectly_affecting, + ) def __str__(self): # TODO - if some of the values is None -> TypeError is raised - return str(self.manufacturer) + ' ' + str(self.name) + ' dgst: ' + self.dgst + return str(self.manufacturer) + " " + str(self.name) + " dgst: " + self.dgst - def merge(self, other: 'CommonCriteriaCert', other_source: Optional[str] = None): + def merge(self, other: "CommonCriteriaCert", other_source: Optional[str] = None): """ Merges with other CC sample. Assuming they come from different sources, e.g., csv and html. Assuming that html source has better protection profiles, they overwrite CSV info @@ -274,31 +334,33 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl """ if self != other: logger.warning( - f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}') + f"Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}" + ) for att, val in vars(self).items(): if not val: setattr(self, att, getattr(other, att)) - elif other_source == 'html' and att == 'protection_profiles': + elif other_source == "html" and att == "protection_profiles": setattr(self, att, getattr(other, att)) - elif other_source == 'html' and att == 'maintenance_updates': + elif other_source == "html" and att == "maintenance_updates": setattr(self, att, getattr(other, att)) - elif att == 'state': + elif att == "state": setattr(self, att, getattr(other, att)) else: if getattr(self, att) != getattr(other, att): logger.warning( - f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}') + f"When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}" + ) @classmethod - def from_dict(cls, dct: Dict) -> 'CommonCriteriaCert': + def from_dict(cls, dct: Dict) -> "CommonCriteriaCert": new_dct = dct.copy() - new_dct['maintenance_updates'] = set(dct['maintenance_updates']) - new_dct['protection_profiles'] = set(dct['protection_profiles']) + new_dct["maintenance_updates"] = set(dct["maintenance_updates"]) + new_dct["protection_profiles"] = set(dct["protection_profiles"]) return super(cls, CommonCriteriaCert).from_dict(new_dct) @classmethod - def from_html_row(cls, row: Tag, status: str, category: str) -> 'CommonCriteriaCert': + def from_html_row(cls, row: Tag, status: str, category: str) -> "CommonCriteriaCert": """ Creates a CC sample from html row """ @@ -319,74 +381,72 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return set(cell.stripped_strings) def _get_manufacturer_web(cell: Tag) -> Optional[str]: - for link in cell.find_all('a'): - if link is not None and link.get('title') == 'Vendor\'s web site' and link.get('href') != 'http://': - return link.get('href') + for link in cell.find_all("a"): + if link is not None and link.get("title") == "Vendor's web site" and link.get("href") != "http://": + return link.get("href") return None def _get_protection_profiles(cell: Tag) -> set: protection_profiles = set() - for link in list(cell.find_all('a')): - if link.get('href') is not None and '/ppfiles/' in link.get('href'): - protection_profiles.add(ProtectionProfile(str(link.contents[0]), CommonCriteriaCert.cc_url + link.get('href'))) + for link in list(cell.find_all("a")): + if link.get("href") is not None and "/ppfiles/" in link.get("href"): + protection_profiles.add( + ProtectionProfile(str(link.contents[0]), CommonCriteriaCert.cc_url + link.get("href")) + ) return protection_profiles def _get_date(cell: Tag) -> Optional[date]: text = cell.get_text() - extracted_date = datetime.strptime( - text, '%Y-%m-%d').date() if text else None + extracted_date = datetime.strptime(text, "%Y-%m-%d").date() if text else None return extracted_date def _get_report_st_links(cell: Tag) -> Tuple[str, str]: - links = cell.find_all('a') + links = cell.find_all("a") # TODO: Exception checks - assert links[1].get('title').startswith('Certification Report') - assert links[2].get('title').startswith('Security Target') + assert links[1].get("title").startswith("Certification Report") + assert links[2].get("title").startswith("Security Target") - report_link = CommonCriteriaCert.cc_url + links[1].get('href') - security_target_link = CommonCriteriaCert.cc_url + \ - links[2].get('href') + report_link = CommonCriteriaCert.cc_url + links[1].get("href") + security_target_link = CommonCriteriaCert.cc_url + links[2].get("href") return report_link, security_target_link def _get_cert_link(cell: Tag) -> Optional[str]: - links = cell.find_all('a') - return CommonCriteriaCert.cc_url + links[0].get('href') if links else None + links = cell.find_all("a") + return CommonCriteriaCert.cc_url + links[0].get("href") if links else None def _get_maintenance_div(cell: Tag) -> Optional[Tag]: - divs = cell.find_all('div') + divs = cell.find_all("div") for d in divs: - if d.find('div') and d.stripped_strings and list(d.stripped_strings)[0] == 'Maintenance Report(s)': + if d.find("div") and d.stripped_strings and list(d.stripped_strings)[0] == "Maintenance Report(s)": return d return None def _get_maintenance_updates(main_div: Tag) -> set: - possible_updates = list(main_div.find_all('li')) + possible_updates = list(main_div.find_all("li")) maintenance_updates = set() for u in possible_updates: text = list(u.stripped_strings)[0] - main_date = datetime.strptime(text.split( - ' ')[0], '%Y-%m-%d').date() if text else None - main_title = text.split('– ')[1] + main_date = datetime.strptime(text.split(" ")[0], "%Y-%m-%d").date() if text else None + main_title = text.split("– ")[1] main_report_link = None main_st_link = None - links = u.find_all('a') - for l in links: - if l.get('title').startswith('Maintenance Report:'): - main_report_link = CommonCriteriaCert.cc_url + \ - l.get('href') - elif l.get('title').startswith('Maintenance ST'): - main_st_link = CommonCriteriaCert.cc_url + \ - l.get('href') + links = u.find_all("a") + for link in links: + if link.get("title").startswith("Maintenance Report:"): + main_report_link = CommonCriteriaCert.cc_url + link.get("href") + elif link.get("title").startswith("Maintenance ST"): + main_st_link = CommonCriteriaCert.cc_url + link.get("href") else: - logger.error('Unknown link in Maintenance part!') + logger.error("Unknown link in Maintenance part!") maintenance_updates.add( - CommonCriteriaCert.MaintenanceReport(main_date, main_title, main_report_link, main_st_link)) + CommonCriteriaCert.MaintenanceReport(main_date, main_title, main_report_link, main_st_link) + ) return maintenance_updates - cells = list(row.find_all('td')) + cells = list(row.find_all("td")) if len(cells) != 7: - logger.error('Unexpected number of cells in CC html row.') + logger.error("Unexpected number of cells in CC html row.") raise name = _get_name(cells[0]) @@ -401,37 +461,54 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl cert_link = _get_cert_link(cells[2]) maintenance_div = _get_maintenance_div(cells[0]) - maintenances = _get_maintenance_updates( - maintenance_div) if maintenance_div else set() + maintenances = _get_maintenance_updates(maintenance_div) if maintenance_div else set() - return cls(status, category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, - report_link, - st_link, cert_link, manufacturer_web, protection_profiles, maintenances, None, None, None) + return cls( + status, + category, + name, + manufacturer, + scheme, + security_level, + not_valid_before, + not_valid_after, + report_link, + st_link, + cert_link, + manufacturer_web, + protection_profiles, + maintenances, + None, + None, + None, + ) - def set_local_paths(self, - report_pdf_dir: Optional[Union[str, Path]], - st_pdf_dir: Optional[Union[str, Path]], - report_txt_dir: Optional[Union[str, Path]], - st_txt_dir: Optional[Union[str, Path]]): + def set_local_paths( + self, + report_pdf_dir: Optional[Union[str, Path]], + st_pdf_dir: Optional[Union[str, Path]], + report_txt_dir: Optional[Union[str, Path]], + st_txt_dir: Optional[Union[str, Path]], + ): if report_pdf_dir is not None: - self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + '.pdf') + self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf") if st_pdf_dir is not None: - self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + '.pdf') + self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf") if report_txt_dir is not None: - self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + '.txt') + self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + ".txt") if st_txt_dir is not None: - self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + '.txt') + self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + ".txt") @staticmethod - def download_pdf_report(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def download_pdf_report(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": exit_code: Union[str, int] if not cert.report_link: - exit_code = 'No link' + exit_code = "No link" else: exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path) if exit_code != requests.codes.ok: - error_msg = f'failed to download report from {cert.report_link}, code: {exit_code}' - logger.error(f'Cert dgst: {cert.dgst} ' + error_msg) + error_msg = f"failed to download report from {cert.report_link}, code: {exit_code}" + logger.error(f"Cert dgst: {cert.dgst} " + error_msg) cert.state.report_download_ok = False if not cert.state.errors: cert.state.errors = [] @@ -439,15 +516,15 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def download_pdf_target(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def download_pdf_target(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": exit_code: Union[str, int] if not cert.st_link: - exit_code = 'No link' + exit_code = "No link" else: exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path) if exit_code != requests.codes.ok: - error_msg = f'failed to download ST from {cert.report_link}, code: {exit_code}' - logger.error(f'Cert dgst: {cert.dgst}' + error_msg) + error_msg = f"failed to download ST from {cert.report_link}, code: {exit_code}" + logger.error(f"Cert dgst: {cert.dgst}" + error_msg) cert.state.st_download_ok = False if not cert.state.errors: cert.state.errors = [] @@ -455,11 +532,11 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def convert_report_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': - exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path, ['-raw']) + def convert_report_pdf(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": + exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path, ["-raw"]) if exit_code != constants.RETURNCODE_OK: - error_msg = 'failed to convert report pdf->txt' - logger.error(f'Cert dgst: {cert.dgst}' + error_msg) + error_msg = "failed to convert report pdf->txt" + logger.error(f"Cert dgst: {cert.dgst}" + error_msg) cert.state.report_convert_ok = False if not cert.state.errors: cert.state.errors = [] @@ -467,11 +544,11 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def convert_target_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': - exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path, ['-raw']) + def convert_target_pdf(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": + exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path, ["-raw"]) if exit_code != constants.RETURNCODE_OK: - error_msg = 'failed to convert security target pdf->txt' - logger.error(f'Cert dgst: {cert.dgst}' + error_msg) + error_msg = "failed to convert security target pdf->txt" + logger.error(f"Cert dgst: {cert.dgst}" + error_msg) cert.state.st_convert_ok = False if not cert.state.errors: cert.state.errors = [] @@ -479,7 +556,7 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def extract_st_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def extract_st_pdf_metadata(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": response, cert.pdf_data.st_metadata = helpers.extract_pdf_metadata(cert.state.st_pdf_path) if response != constants.RETURNCODE_OK: cert.state.st_extract_ok = False @@ -489,7 +566,7 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def extract_report_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def extract_report_pdf_metadata(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": response, cert.pdf_data.report_metadata = helpers.extract_pdf_metadata(cert.state.report_pdf_path) if response != constants.RETURNCODE_OK: cert.state.report_extract_ok = False @@ -499,11 +576,11 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def extract_st_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def extract_st_pdf_frontpage(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": cert.pdf_data.st_frontpage = dict() - response_anssi, cert.pdf_data.st_frontpage['anssi'] = helpers.search_only_headers_anssi(cert.state.st_txt_path) - response_bsi, cert.pdf_data.st_frontpage['bsi'] = helpers.search_only_headers_bsi(cert.state.st_txt_path) + response_anssi, cert.pdf_data.st_frontpage["anssi"] = helpers.search_only_headers_anssi(cert.state.st_txt_path) + response_bsi, cert.pdf_data.st_frontpage["bsi"] = helpers.search_only_headers_bsi(cert.state.st_txt_path) if response_anssi != constants.RETURNCODE_OK: cert.state.st_extract_ok = False @@ -519,12 +596,14 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def extract_report_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def extract_report_pdf_frontpage(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": cert.pdf_data.report_frontpage = dict() - response_bsi, cert.pdf_data.report_frontpage['bsi'] = helpers.search_only_headers_bsi( - cert.state.report_txt_path) - response_anssi, cert.pdf_data.report_frontpage['anssi'] = helpers.search_only_headers_anssi( - cert.state.report_txt_path) + response_bsi, cert.pdf_data.report_frontpage["bsi"] = helpers.search_only_headers_bsi( + cert.state.report_txt_path + ) + response_anssi, cert.pdf_data.report_frontpage["anssi"] = helpers.search_only_headers_anssi( + cert.state.report_txt_path + ) if response_anssi != constants.RETURNCODE_OK: cert.state.report_extract_ok = False @@ -540,14 +619,14 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl return cert @staticmethod - def extract_report_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def extract_report_pdf_keywords(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": response, cert.pdf_data.report_keywords = helpers.extract_keywords(cert.state.report_txt_path) if response != constants.RETURNCODE_OK: cert.state.report_extract_ok = False return cert @staticmethod - def extract_st_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + def extract_st_pdf_keywords(cert: "CommonCriteriaCert") -> "CommonCriteriaCert": response, cert.pdf_data.st_keywords = helpers.extract_keywords(cert.state.st_txt_path) if response != constants.RETURNCODE_OK: cert.state.st_extract_ok = False @@ -561,19 +640,19 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl def compute_heuristics_cpe_vendors(self, cpe_classifier: CPEClassifier): # TODO: This method probably can be deleted. - self.heuristics.cpe_candidate_vendors = cpe_classifier.get_candidate_list_of_vendors(self.manufacturer) # type: ignore + self.heuristics.cpe_candidate_vendors = cpe_classifier.get_candidate_list_of_vendors(self.manufacturer) # type: ignore def compute_heuristics_cpe_match(self, cpe_classifier: CPEClassifier): self.heuristics.cpe_matches = cpe_classifier.predict_single_cert(self.manufacturer, self.name, self.heuristics.extracted_versions) # type: ignore def compute_heuristics_cert_lab(self): if not self.pdf_data: - logger.error('Cannot compute sample lab when pdf files were not processed.') + logger.error("Cannot compute sample lab when pdf files were not processed.") return self.heuristics.cert_lab = self.pdf_data.cert_lab def compute_heuristics_cert_id(self): if not self.pdf_data: - logger.error('Cannot compute sample id when pdf files were not processed.') + logger.error("Cannot compute sample id when pdf files were not processed.") return self.heuristics.cert_id = self.pdf_data.cert_id diff --git a/sec_certs/sample/cpe.py b/sec_certs/sample/cpe.py index ea1510b3..aeab6dc9 100644 --- a/sec_certs/sample/cpe.py +++ b/sec_certs/sample/cpe.py @@ -1,12 +1,13 @@ from dataclasses import dataclass -from typing import ClassVar, List, Optional, Tuple, Dict +from typing import ClassVar, Dict, List, Optional, Tuple + from sec_certs.serialization.json import ComplexSerializableType from sec_certs.serialization.pandas import PandasSerializableType @dataclass(init=False) class CPE(PandasSerializableType, ComplexSerializableType): - uri: Optional[str] + uri: str title: Optional[str] version: Optional[str] vendor: Optional[str] @@ -14,50 +15,61 @@ class CPE(PandasSerializableType, ComplexSerializableType): start_version: Optional[Tuple[str, str]] end_version: Optional[Tuple[str, str]] - pandas_columns: ClassVar[List[str]] = ['uri', 'vendor', 'item_name', 'version', 'title', 'start_version', 'end_version'] + pandas_columns: ClassVar[List[str]] = [ + "uri", + "vendor", + "item_name", + "version", + "title", + "start_version", + "end_version", + ] - def __init__(self, uri: Optional[str] = None, - title: Optional[str] = None, - start_version: Optional[Tuple[str, str]] = None, - end_version: Optional[Tuple[str, str]] = None): + def __init__( + self, + uri: str, + title: Optional[str] = None, + start_version: Optional[Tuple[str, str]] = None, + end_version: Optional[Tuple[str, str]] = None, + ): self.uri = uri self.title = title self.start_version = start_version self.end_version = end_version if self.uri: - self.vendor = ' '.join(self.uri.split(':')[3].split('_')) - self.item_name = ' '.join(self.uri.split(':')[4].split('_')) - self.version = self.uri.split(':')[5] + self.vendor = " ".join(self.uri.split(":")[3].split("_")) + self.item_name = " ".join(self.uri.split(":")[4].split("_")) + self.version = self.uri.split(":")[5] - def __lt__(self, other: 'CPE'): + def __lt__(self, other: "CPE"): if self.title is None or other.title is None: raise RuntimeError("Cannot compare CPEs because title is missing.") return self.title < other.title @classmethod def from_dict(cls, dct: Dict): - if isinstance(dct['start_version'], list): - dct['start_version'] = tuple(dct['start_version']) - if isinstance(dct['end_version'], list): - dct['end_version'] = tuple(dct['end_version']) + if isinstance(dct["start_version"], list): + dct["start_version"] = tuple(dct["start_version"]) + if isinstance(dct["end_version"], list): + dct["end_version"] = tuple(dct["end_version"]) return super().from_dict(dct) @property def serialized_attributes(self) -> List[str]: - return ['uri', 'title', 'start_version', 'end_version'] + return ["uri", "title", "start_version", "end_version"] @property def update(self) -> str: if self.uri is None: raise RuntimeError("URI is missing.") - return ' '.join(self.uri.split(':')[6].split('_')) + return " ".join(self.uri.split(":")[6].split("_")) @property def target_hw(self) -> str: if self.uri is None: raise RuntimeError("URI is missing.") - return ' '.join(self.uri.split(':')[11].split('_')) + return " ".join(self.uri.split(":")[11].split("_")) @property def pandas_tuple(self): @@ -67,4 +79,9 @@ class CPE(PandasSerializableType, ComplexSerializableType): return hash((self.uri, self.start_version, self.end_version)) def __eq__(self, other): - return isinstance(other, self.__class__) and self.uri == other.uri and self.start_version == other.start_version and self.end_version == other.end_version
\ No newline at end of file + return ( + isinstance(other, self.__class__) + and self.uri == other.uri + and self.start_version == other.start_version + and self.end_version == other.end_version + ) diff --git a/sec_certs/sample/cve.py b/sec_certs/sample/cve.py index f63a8869..2bc28c4a 100644 --- a/sec_certs/sample/cve.py +++ b/sec_certs/sample/cve.py @@ -1,13 +1,13 @@ import datetime import itertools from dataclasses import dataclass -from typing import Any, Dict, List, Optional, ClassVar, Tuple +from typing import ClassVar, Dict, List, Optional, Tuple from dateutil.parser import isoparse +from sec_certs.sample.cpe import CPE from sec_certs.serialization.json import ComplexSerializableType from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.sample.cpe import CPE @dataclass(init=False) @@ -24,26 +24,37 @@ class CVE(PandasSerializableType, ComplexSerializableType): """ Will load Impact from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1 """ - if not dct['impact']: - return cls(0, '', 0, 0) - elif 'baseMetricV3' in dct['impact']: - return cls(dct['impact']['baseMetricV3']['cvssV3']['baseScore'], - dct['impact']['baseMetricV3']['cvssV3']['baseSeverity'], - dct['impact']['baseMetricV3']['exploitabilityScore'], - dct['impact']['baseMetricV3']['impactScore']) - elif 'baseMetricV2' in dct['impact']: - return cls(dct['impact']['baseMetricV2']['cvssV2']['baseScore'], - dct['impact']['baseMetricV2']['severity'], - dct['impact']['baseMetricV2']['exploitabilityScore'], - dct['impact']['baseMetricV2']['impactScore']) + if not dct["impact"]: + return cls(0, "", 0, 0) + elif "baseMetricV3" in dct["impact"]: + return cls( + dct["impact"]["baseMetricV3"]["cvssV3"]["baseScore"], + dct["impact"]["baseMetricV3"]["cvssV3"]["baseSeverity"], + dct["impact"]["baseMetricV3"]["exploitabilityScore"], + dct["impact"]["baseMetricV3"]["impactScore"], + ) + elif "baseMetricV2" in dct["impact"]: + return cls( + dct["impact"]["baseMetricV2"]["cvssV2"]["baseScore"], + dct["impact"]["baseMetricV2"]["severity"], + dct["impact"]["baseMetricV2"]["exploitabilityScore"], + dct["impact"]["baseMetricV2"]["impactScore"], + ) cve_id: str vulnerable_cpes: List[CPE] impact: Impact published_date: Optional[datetime.datetime] - pandas_columns: ClassVar[List[str]] = ['cve_id', 'vulnerable_cpes', 'base_score', 'severity', - 'explotability_score', 'impact_score', 'published_date'] + pandas_columns: ClassVar[List[str]] = [ + "cve_id", + "vulnerable_cpes", + "base_score", + "severity", + "explotability_score", + "impact_score", + "published_date", + ] def __init__(self, cve_id: str, vulnerable_cpes: List[CPE], impact: Impact, published_date: str): self.cve_id = cve_id @@ -61,48 +72,56 @@ class CVE(PandasSerializableType, ComplexSerializableType): def __lt__(self, other): if not isinstance(other, CVE): - raise ValueError(f'Cannot compare CVE with {type(other)} type.') - self_year = int(self.cve_id.split('-')[1]) - self_id = int(self.cve_id.split('-')[2]) - other_year = int(other.cve_id.split('-')[1]) - other_id = int(other.cve_id.split('-')[2]) + raise ValueError(f"Cannot compare CVE with {type(other)} type.") + self_year = int(self.cve_id.split("-")[1]) + self_id = int(self.cve_id.split("-")[2]) + other_year = int(other.cve_id.split("-")[1]) + other_id = int(other.cve_id.split("-")[2]) return self_year < other_year if self_year != other_year else self_id < other_id @property def pandas_tuple(self): - return (self.cve_id, self.vulnerable_cpes, self.impact.base_score, self.impact.severity, - self.impact.explotability_score, self.impact.impact_score, self.published_date) + return ( + self.cve_id, + self.vulnerable_cpes, + self.impact.base_score, + self.impact.severity, + self.impact.explotability_score, + self.impact.impact_score, + self.published_date, + ) @classmethod - def from_nist_dict(cls, dct: Dict) -> 'CVE': + def from_nist_dict(cls, dct: Dict) -> "CVE": """ Will load CVE from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1 """ + def get_vulnerable_cpes_from_nist_dict(dct: Dict) -> List[CPE]: def get_vulnerable_cpes_from_node(node: Dict) -> List[CPE]: cpe_uris = [] - if 'children' in node: - for child in node['children']: + if "children" in node: + for child in node["children"]: cpe_uris += get_vulnerable_cpes_from_node(child) - if 'cpe_match' in node: - lst = node['cpe_match'] + if "cpe_match" in node: + lst = node["cpe_match"] for x in lst: - if x['vulnerable']: - cpe_uri = x['cpe23Uri'] + if x["vulnerable"]: + cpe_uri = x["cpe23Uri"] version_start: Optional[Tuple[str, str]] version_end: Optional[Tuple[str, str]] - if 'versionStartIncluding' in x and x['versionStartIncluding']: - version_start = ('including', x['versionStartIncluding']) - elif 'versionStartExcluding' in x and x['versionStartExcluding']: - version_start = ('excluding', x['versionStartExcluding']) + if "versionStartIncluding" in x and x["versionStartIncluding"]: + version_start = ("including", x["versionStartIncluding"]) + elif "versionStartExcluding" in x and x["versionStartExcluding"]: + version_start = ("excluding", x["versionStartExcluding"]) else: version_start = None - if 'versionEndIncluding' in x and x['versionEndIncluding']: - version_end = ('including', x['versionEndIncluding']) - elif 'versionEndExcluding' in x and x['versionEndExcluding']: - version_end = ('excluding', x['versionEndExcluding']) + if "versionEndIncluding" in x and x["versionEndIncluding"]: + version_end = ("including", x["versionEndIncluding"]) + elif "versionEndExcluding" in x and x["versionEndExcluding"]: + version_end = ("excluding", x["versionEndExcluding"]) else: version_end = None @@ -110,11 +129,15 @@ class CVE(PandasSerializableType, ComplexSerializableType): return cpe_uris - return list(itertools.chain.from_iterable([get_vulnerable_cpes_from_node(x) for x in dct['configurations']['nodes']])) + return list( + itertools.chain.from_iterable( + [get_vulnerable_cpes_from_node(x) for x in dct["configurations"]["nodes"]] + ) + ) - cve_id = dct['cve']['CVE_data_meta']['ID'] + cve_id = dct["cve"]["CVE_data_meta"]["ID"] impact = cls.Impact.from_nist_dict(dct) vulnerable_cpes = get_vulnerable_cpes_from_nist_dict(dct) - published_date = dct['publishedDate'] + published_date = dct["publishedDate"] return cls(cve_id, vulnerable_cpes, impact, published_date) diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py index a8842e00..b66d11df 100644 --- a/sec_certs/sample/fips.py +++ b/sec_certs/sample/fips.py @@ -3,31 +3,31 @@ import re from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import ClassVar, Dict, Optional, Union, List, Tuple, Set, Pattern +from typing import ClassVar, Dict, List, Optional, Pattern, Set, Tuple, Union import requests -from bs4 import Tag, NavigableString, BeautifulSoup +from bs4 import BeautifulSoup, NavigableString, Tag from dateutil import parser from tabula import read_pdf -import sec_certs.constants -from sec_certs import helpers, constants as constants -from sec_certs.cert_rules import fips_common_rules, REGEXEC_SEP, fips_rules - -from sec_certs.sample.certificate import Certificate, logger +import sec_certs.constants as constants +from sec_certs import helpers +from sec_certs.cert_rules import REGEXEC_SEP, fips_common_rules, fips_rules from sec_certs.config.configuration import config from sec_certs.constants import LINE_SEPARATOR -from sec_certs.helpers import save_modified_cert_file, normalize_match_string, load_cert_file -from sec_certs.serialization.json import ComplexSerializableType from sec_certs.dataset.cpe import CPEDataset -from sec_certs.sample.cpe import CPE +from sec_certs.helpers import load_cert_file, normalize_match_string, save_modified_cert_file from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.sample.certificate import Certificate, logger +from sec_certs.sample.cpe import CPE +from sec_certs.serialization.json import ComplexSerializableType class FIPSCertificate(Certificate, ComplexSerializableType): - FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov' + FIPS_BASE_URL: ClassVar[str] = "https://csrc.nist.gov" FIPS_MODULE_URL: ClassVar[ - str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/' + str + ] = "https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/" @dataclass(eq=True) class State(ComplexSerializableType): @@ -38,23 +38,34 @@ class FIPSCertificate(Certificate, ComplexSerializableType): file_status: Optional[bool] txt_state: bool - def __init__(self, sp_path: Union[str, Path], html_path: Union[str, Path], fragment_path: Union[str, Path], - tables_done: bool, file_status: Optional[bool], txt_state: bool): + def __init__( + self, + sp_path: Union[str, Path], + html_path: Union[str, Path], + fragment_path: Union[str, Path], + tables_done: bool, + file_status: Optional[bool], + txt_state: bool, + ): self.sp_path = Path(sp_path) self.html_path = Path(html_path) self.fragment_path = Path(fragment_path) self.tables_done = tables_done self.file_status = file_status self.txt_state = txt_state - - def set_local_paths(self, sp_dir: Optional[Union[str, Path]], html_dir: Optional[Union[str, Path]], - fragment_dir: Optional[Union[str, Path]]): + + def set_local_paths( + self, + sp_dir: Optional[Union[str, Path]], + html_dir: Optional[Union[str, Path]], + fragment_dir: Optional[Union[str, Path]], + ): if sp_dir is not None: - self.state.sp_path = (Path(sp_dir) / (self.dgst)).with_suffix('.pdf') + self.state.sp_path = (Path(sp_dir) / (self.dgst)).with_suffix(".pdf") if html_dir is not None: self.state.html_path = (Path(html_dir) / (self.dgst)).with_suffix(".html") if fragment_dir is not None: - self.state.fragment_path = (Path(fragment_dir) / (self.dgst)).with_suffix('.txt') + self.state.fragment_path = (Path(fragment_dir) / (self.dgst)).with_suffix(".txt") @dataclass(eq=True) class Algorithm(ComplexSerializableType): @@ -71,10 +82,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): return self.type def __repr__(self): - return self.type + ' algorithm #' + self.cert_id + ' created by ' + self.vendor + return self.type + " algorithm #" + self.cert_id + " created by " + self.vendor def __str__(self): - return str(self.type + ' algorithm #' + self.cert_id + ' created by ' + self.vendor) + return str(self.type + " algorithm #" + self.cert_id + " created by " + self.vendor) @dataclass(eq=True) class WebScan(ComplexSerializableType): @@ -108,10 +119,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): connections: List[str] def __post_init__(self): - self.date_validation = [parser.parse(x).date() for x in - self.date_validation] if self.date_validation else None - self.date_sunset = parser.parse( - self.date_sunset).date() if self.date_sunset else None + self.date_validation = ( + [parser.parse(x).date() for x in self.date_validation] if self.date_validation else None + ) + self.date_sunset = parser.parse(self.date_sunset).date() if self.date_sunset else None @property def dgst(self): @@ -120,10 +131,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): return helpers.get_first_16_bytes_sha256(self.product_url + self.vendor_www) def __repr__(self): - return self.module_name + ' created by ' + self.vendor + return self.module_name + " created by " + self.vendor def __str__(self): - return str(self.module_name + ' created by ' + self.vendor) + return str(self.module_name + " created by " + self.vendor) @dataclass(eq=True) class PdfScan(ComplexSerializableType): @@ -165,7 +176,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): @property def serialized_attributes(self) -> List[str]: all_vars = copy.deepcopy(super().serialized_attributes) - all_vars.remove('cpe_candidate_vendors') + all_vars.remove("cpe_candidate_vendors") return all_vars def __post_init__(self): @@ -186,23 +197,34 @@ class FIPSCertificate(Certificate, ComplexSerializableType): @property def label_studio_title(self): - return 'Vendor: ' + str(self.web_scan.vendor) + '\n' \ - + 'Module name: ' + str(self.web_scan.module_name) + '\n' \ - + 'HW version: ' + str(self.web_scan.hw_version) + '\n' \ - + 'FW version: ' + str(self.web_scan.fw_version) + return ( + "Vendor: " + + str(self.web_scan.vendor) + + "\n" + + "Module name: " + + str(self.web_scan.module_name) + + "\n" + + "HW version: " + + str(self.web_scan.hw_version) + + "\n" + + "FW version: " + + str(self.web_scan.fw_version) + ) @staticmethod def download_security_policy(cert: Tuple[str, Path]) -> None: exit_code = helpers.download_file(*cert, delay=1) if exit_code != requests.codes.ok: - logger.error( - f'Failed to download security policy from {cert[0]}, code: {exit_code}') + logger.error(f"Failed to download security policy from {cert[0]}, code: {exit_code}") - def __init__(self, cert_id: str, - web_scan: 'FIPSCertificate.WebScan', - pdf_scan: 'FIPSCertificate.PdfScan', - heuristics: 'FIPSCertificate.FIPSHeuristics', - state: State): + def __init__( + self, + cert_id: str, + web_scan: "FIPSCertificate.WebScan", + pdf_scan: "FIPSCertificate.PdfScan", + heuristics: "FIPSCertificate.FIPSHeuristics", + state: State, + ): super().__init__() self.cert_id = cert_id self.web_scan = web_scan @@ -214,20 +236,42 @@ class FIPSCertificate(Certificate, ComplexSerializableType): def download_html_page(cert: Tuple[str, Path]) -> Optional[Tuple[str, Path]]: exit_code = helpers.download_file(*cert, delay=1) if exit_code != requests.codes.ok: - logger.error( - f'Failed to download html page from {cert[0]}, code: {exit_code}') + logger.error(f"Failed to download html page from {cert[0]}, code: {exit_code}") return cert return None @staticmethod def initialize_dictionary() -> Dict: - return {'module_name': None, 'standard': None, 'status': None, 'date_sunset': None, - 'date_validation': None, 'level': None, 'caveat': None, 'exceptions': None, - 'type': None, 'embodiment': None, 'tested_conf': None, 'description': None, - 'vendor': None, 'vendor_www': None, 'lab': None, 'lab_nvlap': None, - 'historical_reason': None, 'revoked_reason': None, 'revoked_link': None, 'algorithms': [], - 'mentioned_certs': {}, 'tables_done': False, 'security_policy_www': None, 'certificate_www': None, - 'hw_versions': None, 'fw_versions': None, 'sw_versions': None, 'product_url': None} + return { + "module_name": None, + "standard": None, + "status": None, + "date_sunset": None, + "date_validation": None, + "level": None, + "caveat": None, + "exceptions": None, + "type": None, + "embodiment": None, + "tested_conf": None, + "description": None, + "vendor": None, + "vendor_www": None, + "lab": None, + "lab_nvlap": None, + "historical_reason": None, + "revoked_reason": None, + "revoked_link": None, + "algorithms": [], + "mentioned_certs": {}, + "tables_done": False, + "security_policy_www": None, + "certificate_www": None, + "hw_versions": None, + "fw_versions": None, + "sw_versions": None, + "product_url": None, + } @staticmethod def parse_caveat(current_text: str) -> Dict[str, Dict[str, int]]: @@ -239,12 +283,12 @@ class FIPSCertificate(Certificate, ComplexSerializableType): ids_found: Dict[str, Dict[str, int]] = {} r_key = r"(?P<word>\w+)?\s?(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)+(?P<id>\d+)" for m in re.finditer(r_key, current_text): - if m.group('word') and m.group('word').lower() in {'rsa', 'shs', 'dsa', 'pkcs', 'aes'}: + if m.group("word") and m.group("word").lower() in {"rsa", "shs", "dsa", "pkcs", "aes"}: continue - if m.group('id') in ids_found: - ids_found[m.group('id')]['count'] += 1 + if m.group("id") in ids_found: + ids_found[m.group("id")]["count"] += 1 else: - ids_found[m.group('id')] = {'count': 1} + ids_found[m.group("id")] = {"count": 1} return ids_found @@ -274,129 +318,135 @@ class FIPSCertificate(Certificate, ComplexSerializableType): :return: list of all found algorithm IDs """ found_items = [] - trs = element.find_all('tr') + trs = element.find_all("tr") for tr in trs: - tds = tr.find_all('td') + tds = tr.find_all("td") cert = FIPSCertificate.extract_algorithm_certificates(tds[1].text) found_items.append( - {'Name': tds[0].text, - 'Certificate': cert[0]['Certificate'] if cert != [] else [], - 'Links': [str(x) for x in tds[1].find_all('a')], - 'Raw': str(tr)}) + { + "Name": tds[0].text, + "Certificate": cert[0]["Certificate"] if cert != [] else [], + "Links": [str(x) for x in tds[1].find_all("a")], + "Raw": str(tr), + } + ) return found_items @staticmethod def parse_html_main(current_div: Tag, html_items_found: Dict, pairs: Dict): - title = current_div.find('div', class_='col-md-3').text.strip() - content = current_div.find('div', class_='col-md-9').text.strip() \ - .replace('\n', '').replace('\t', '').replace(' ', ' ') + title = current_div.find("div", class_="col-md-3").text.strip() + content = ( + current_div.find("div", class_="col-md-9") + .text.strip() + .replace("\n", "") + .replace("\t", "") + .replace(" ", " ") + ) if title in pairs: - if 'date_validation' == pairs[title]: - html_items_found[pairs[title]] = [ - x for x in content.split(';')] + if "date_validation" == pairs[title]: + html_items_found[pairs[title]] = [x for x in content.split(";")] - elif 'caveat' in pairs[title]: + elif "caveat" in pairs[title]: html_items_found[pairs[title]] = content - html_items_found['mentioned_certs'].update(FIPSCertificate.parse_caveat(content)) + html_items_found["mentioned_certs"].update(FIPSCertificate.parse_caveat(content)) - elif 'FIPS Algorithms' in title: - html_items_found['algorithms'] += FIPSCertificate.parse_table( - current_div.find('div', class_='col-md-9')) + elif "FIPS Algorithms" in title: + html_items_found["algorithms"] += FIPSCertificate.parse_table( + current_div.find("div", class_="col-md-9") + ) - elif 'Algorithms' in title or 'Description' in title: - html_items_found['algorithms'] += FIPSCertificate.extract_algorithm_certificates( - content) - if 'Description' in title: - html_items_found['description'] = content + elif "Algorithms" in title or "Description" in title: + html_items_found["algorithms"] += FIPSCertificate.extract_algorithm_certificates(content) + if "Description" in title: + html_items_found["description"] = content - elif 'tested_conf' in pairs[title] or 'exceptions' in pairs[title]: - html_items_found[pairs[title]] = [x.text for x in - current_div.find('div', class_='col-md-9').find_all('li')] + elif "tested_conf" in pairs[title] or "exceptions" in pairs[title]: + html_items_found[pairs[title]] = [ + x.text for x in current_div.find("div", class_="col-md-9").find_all("li") + ] else: html_items_found[pairs[title]] = content @staticmethod def parse_vendor(current_div: Tag, html_items_found: Dict, current_file: Path): - vendor_string = current_div.find('div', 'panel-body').find('a') + vendor_string = current_div.find("div", "panel-body").find("a") if not vendor_string: - vendor_string = list(current_div.find( - 'div', 'panel-body').children)[0].strip() - html_items_found['vendor_www'] = '' + vendor_string = list(current_div.find("div", "panel-body").children)[0].strip() + html_items_found["vendor_www"] = "" else: - html_items_found['vendor_www'] = vendor_string.get('href') + html_items_found["vendor_www"] = vendor_string.get("href") vendor_string = vendor_string.text.strip() - html_items_found['vendor'] = vendor_string - if html_items_found['vendor'] == '': + html_items_found["vendor"] = vendor_string + if html_items_found["vendor"] == "": logger.warning(f"NO VENDOR FOUND {current_file}") @staticmethod def parse_lab(current_div: Tag, html_items_found: Dict, current_file: Path): - html_items_found['lab'] = list( - current_div.find('div', 'panel-body').children)[0].strip() - html_items_found['nvlap_code'] = \ - list(current_div.find( - 'div', 'panel-body').children)[2].strip().split('\n')[1].strip() + html_items_found["lab"] = list(current_div.find("div", "panel-body").children)[0].strip() + html_items_found["nvlap_code"] = ( + list(current_div.find("div", "panel-body").children)[2].strip().split("\n")[1].strip() + ) - if html_items_found['lab'] == '': + if html_items_found["lab"] == "": logger.warning(f"NO LAB FOUND {current_file}") - if html_items_found['nvlap_code'] == '': + if html_items_found["nvlap_code"] == "": logger.warning(f"NO NVLAP CODE FOUND {current_file}") @staticmethod def parse_related_files(current_div: Tag, html_items_found: Dict): - links = current_div.find_all('a') - html_items_found['security_policy_www'] = constants.FIPS_BASE_URL + links[0].get('href') + links = current_div.find_all("a") + html_items_found["security_policy_www"] = constants.FIPS_BASE_URL + links[0].get("href") if len(links) == 2: - html_items_found['certificate_www'] = constants.FIPS_BASE_URL + links[1].get('href') + html_items_found["certificate_www"] = constants.FIPS_BASE_URL + links[1].get("href") @staticmethod def normalize(items: Dict): - items['type'] = items['type'].lower().replace('-', ' ').title() - items['embodiment'] = items['embodiment'].lower().replace( - '-', ' ').replace('stand alone', 'standalone').title() + items["type"] = items["type"].lower().replace("-", " ").title() + items["embodiment"] = items["embodiment"].lower().replace("-", " ").replace("stand alone", "standalone").title() @classmethod - def html_from_file(cls, file: Path, state: State, initialized: 'FIPSCertificate' = None, - redo: bool = False) -> 'FIPSCertificate': + def html_from_file( + cls, file: Path, state: State, initialized: "FIPSCertificate" = None, redo: bool = False + ) -> "FIPSCertificate": pairs = { - 'Module Name': 'module_name', - 'Standard': 'standard', - 'Status': 'status', - 'Sunset Date': 'date_sunset', - 'Validation Dates': 'date_validation', - 'Overall Level': 'level', - 'Caveat': 'caveat', - 'Security Level Exceptions': 'exceptions', - 'Module Type': 'type', - 'Embodiment': 'embodiment', - 'FIPS Algorithms': 'algorithms', - 'Allowed Algorithms': 'algorithms', - 'Other Algorithms': 'algorithms', - 'Tested Configuration(s)': 'tested_conf', - 'Description': 'description', - 'Historical Reason': 'historical_reason', - 'Hardware Versions': 'hw_versions', - 'Firmware Versions': 'fw_versions', - 'Revoked Reason': 'revoked_reason', - 'Revoked Link': 'revoked_link', - 'Software Versions': 'sw_versions', - 'Product URL': 'product_url' + "Module Name": "module_name", + "Standard": "standard", + "Status": "status", + "Sunset Date": "date_sunset", + "Validation Dates": "date_validation", + "Overall Level": "level", + "Caveat": "caveat", + "Security Level Exceptions": "exceptions", + "Module Type": "type", + "Embodiment": "embodiment", + "FIPS Algorithms": "algorithms", + "Allowed Algorithms": "algorithms", + "Other Algorithms": "algorithms", + "Tested Configuration(s)": "tested_conf", + "Description": "description", + "Historical Reason": "historical_reason", + "Hardware Versions": "hw_versions", + "Firmware Versions": "fw_versions", + "Revoked Reason": "revoked_reason", + "Revoked Link": "revoked_link", + "Software Versions": "sw_versions", + "Product URL": "product_url", } if not initialized: items_found = FIPSCertificate.initialize_dictionary() - items_found['cert_id'] = file.stem + items_found["cert_id"] = file.stem else: items_found = initialized.web_scan.__dict__ - items_found['cert_id'] = initialized.cert_id - items_found['revoked_reason'] = None - items_found['revoked_link'] = None - items_found['mentioned_certs'] = {} + items_found["cert_id"] = initialized.cert_id + items_found["revoked_reason"] = None + items_found["revoked_link"] = None + items_found["mentioned_certs"] = {} state.tables_done = initialized.state.tables_done state.file_status = initialized.state.file_status state.txt_state = initialized.state.txt_state @@ -404,80 +454,79 @@ class FIPSCertificate(Certificate, ComplexSerializableType): if redo: items_found = FIPSCertificate.initialize_dictionary() - items_found['cert_id'] = file.stem + items_found["cert_id"] = file.stem text = helpers.load_cert_html_file(file) - soup = BeautifulSoup(text, 'html.parser') - for div in soup.find_all('div', class_='row padrow'): + soup = BeautifulSoup(text, "html.parser") + for div in soup.find_all("div", class_="row padrow"): FIPSCertificate.parse_html_main(div, items_found, pairs) - for div in soup.find_all('div', class_='panel panel-default')[1:]: - if div.find('h4', class_='panel-title').text == 'Vendor': + for div in soup.find_all("div", class_="panel panel-default")[1:]: + if div.find("h4", class_="panel-title").text == "Vendor": FIPSCertificate.parse_vendor(div, items_found, file) - if div.find('h4', class_='panel-title').text == 'Lab': + if div.find("h4", class_="panel-title").text == "Lab": FIPSCertificate.parse_lab(div, items_found, file) - if div.find('h4', class_='panel-title').text == 'Related Files': + if div.find("h4", class_="panel-title").text == "Related Files": FIPSCertificate.parse_related_files(div, items_found) FIPSCertificate.normalize(items_found) - return FIPSCertificate(items_found['cert_id'], - FIPSCertificate.WebScan( - items_found['module_name'] if 'module_name' in items_found else None, - items_found['standard'] if 'standard' in items_found else None, - items_found['status'] if 'status' in items_found else None, - items_found['date_sunset'] if 'date_sunset' in items_found else None, - items_found['date_validation'] if 'date_validation' in items_found else None, - items_found['level'] if 'level' in items_found else None, - items_found['caveat'] if 'caveat' in items_found else None, - items_found['exceptions'] if 'exceptions' in items_found else None, - items_found['type'] if 'type' in items_found else None, - items_found['embodiment'] if 'embodiment' in items_found else None, - items_found['algorithms'] if 'algorithms' in items_found else None, - items_found['tested_conf'] if 'tested_conf' in items_found else None, - items_found['description'] if 'description' in items_found else None, - items_found['mentioned_certs'] if 'mentioned_certs' in items_found else None, - items_found['vendor'] if 'vendor' in items_found else None, - items_found['vendor_www'] if 'vendor_www' in items_found else None, - items_found['lab'] if 'lab' in items_found else None, - items_found['nvlap_code'] if 'nvlap_code' in items_found else None, - items_found['historical_reason'] if 'historical_reason' in items_found else None, - items_found['security_policy_www'] if 'security_policy_www' in items_found else None, - items_found['certificate_www'] if 'certificate_www' in items_found else None, - items_found['hw_versions'] if 'hw_versions' in items_found else None, - items_found['fw_versions'] if 'fw_versions' in items_found else None, - items_found['revoked_reason'] if 'revoked_reason' in items_found else None, - items_found['revoked_link'] if 'revoked_link' in items_found else None, - items_found['sw_versions'] if 'sw_versions' in items_found else None, - items_found['product_url'] if 'product_url' in items_found else None, - [] - ), # connections - FIPSCertificate.PdfScan( - items_found['cert_id'], - {} if not initialized else initialized.pdf_scan.keywords, - [] if not initialized else initialized.pdf_scan.algorithms, - [] # connections - ), - FIPSCertificate.FIPSHeuristics(None, [], [], 0), - state - ) + return FIPSCertificate( + items_found["cert_id"], + FIPSCertificate.WebScan( + items_found["module_name"] if "module_name" in items_found else None, + items_found["standard"] if "standard" in items_found else None, + items_found["status"] if "status" in items_found else None, + items_found["date_sunset"] if "date_sunset" in items_found else None, + items_found["date_validation"] if "date_validation" in items_found else None, + items_found["level"] if "level" in items_found else None, + items_found["caveat"] if "caveat" in items_found else None, + items_found["exceptions"] if "exceptions" in items_found else None, + items_found["type"] if "type" in items_found else None, + items_found["embodiment"] if "embodiment" in items_found else None, + items_found["algorithms"] if "algorithms" in items_found else None, + items_found["tested_conf"] if "tested_conf" in items_found else None, + items_found["description"] if "description" in items_found else None, + items_found["mentioned_certs"] if "mentioned_certs" in items_found else None, + items_found["vendor"] if "vendor" in items_found else None, + items_found["vendor_www"] if "vendor_www" in items_found else None, + items_found["lab"] if "lab" in items_found else None, + items_found["nvlap_code"] if "nvlap_code" in items_found else None, + items_found["historical_reason"] if "historical_reason" in items_found else None, + items_found["security_policy_www"] if "security_policy_www" in items_found else None, + items_found["certificate_www"] if "certificate_www" in items_found else None, + items_found["hw_versions"] if "hw_versions" in items_found else None, + items_found["fw_versions"] if "fw_versions" in items_found else None, + items_found["revoked_reason"] if "revoked_reason" in items_found else None, + items_found["revoked_link"] if "revoked_link" in items_found else None, + items_found["sw_versions"] if "sw_versions" in items_found else None, + items_found["product_url"] if "product_url" in items_found else None, + [], + ), # connections + FIPSCertificate.PdfScan( + items_found["cert_id"], + {} if not initialized else initialized.pdf_scan.keywords, + [] if not initialized else initialized.pdf_scan.algorithms, + [], # connections + ), + FIPSCertificate.FIPSHeuristics(None, [], [], 0), + state, + ) @staticmethod - def convert_pdf_file(tup: Tuple['FIPSCertificate', Path, Path]) -> 'FIPSCertificate': + def convert_pdf_file(tup: Tuple["FIPSCertificate", Path, Path]) -> "FIPSCertificate": cert, pdf_path, txt_path = tup if not cert.state.txt_state: - exit_code = helpers.convert_pdf_file(pdf_path, txt_path, ['-raw']) + exit_code = helpers.convert_pdf_file(pdf_path, txt_path, ["-raw"]) if exit_code != constants.RETURNCODE_OK: - logger.error( - f'Cert dgst: {cert.dgst} failed to convert security policy pdf->txt') + logger.error(f"Cert dgst: {cert.dgst} failed to convert security policy pdf->txt") cert.state.txt_state = False else: cert.state.txt_state = True return cert - @staticmethod def _declare_state(text: str): """ @@ -486,61 +535,59 @@ class FIPSCertificate(Certificate, ComplexSerializableType): :param text: security policy content :return: True if parsable, otherwise False """ - return len(text) * 0.5 <= len(''.join(filter(str.isalpha, text))) + return len(text) * 0.5 <= len("".join(filter(str.isalpha, text))) @staticmethod - def find_keywords(cert: 'FIPSCertificate') -> Tuple[Optional[Dict], 'FIPSCertificate']: + def find_keywords(cert: "FIPSCertificate") -> Tuple[Optional[Dict], "FIPSCertificate"]: if not cert.state.txt_state: return None, cert - text, text_with_newlines, unicode_error = load_cert_file(cert.state.sp_path.with_suffix('.pdf.txt'), - -1, LINE_SEPARATOR) + text, text_with_newlines, unicode_error = load_cert_file( + cert.state.sp_path.with_suffix(".pdf.txt"), -1, LINE_SEPARATOR + ) text_to_parse = text_with_newlines if config.use_text_with_newlines_during_parsing else text cert.state.txt_state = FIPSCertificate._declare_state(text) if config.ignore_first_page: - text_to_parse = text_to_parse[text_to_parse.index(" "):] + text_to_parse = text_to_parse[text_to_parse.index(" ") :] items_found, fips_text = FIPSCertificate.parse_cert_file(FIPSCertificate.remove_platforms(text_to_parse)) - save_modified_cert_file(cert.state.fragment_path.with_suffix( - '.fips.txt'), fips_text, unicode_error) + save_modified_cert_file(cert.state.fragment_path.with_suffix(".fips.txt"), fips_text, unicode_error) - common_items_found, common_text = FIPSCertificate.parse_cert_file_common(text_to_parse, text_with_newlines, - fips_common_rules) + common_items_found, common_text = FIPSCertificate.parse_cert_file_common( + text_to_parse, text_with_newlines, fips_common_rules + ) - save_modified_cert_file(cert.state.fragment_path.with_suffix( - '.common.txt'), common_text, unicode_error) + save_modified_cert_file(cert.state.fragment_path.with_suffix(".common.txt"), common_text, unicode_error) items_found.update(common_items_found) return items_found, cert @staticmethod - def match_web_algs_to_pdf(cert: 'FIPSCertificate') -> int: - algs_vals = list( - cert.pdf_scan.keywords['rules_fips_algorithms'].values()) - table_vals = [x['Certificate'] for x in cert.pdf_scan.algorithms] + def match_web_algs_to_pdf(cert: "FIPSCertificate") -> int: + algs_vals = list(cert.pdf_scan.keywords["rules_fips_algorithms"].values()) + table_vals = [x["Certificate"] for x in cert.pdf_scan.algorithms] tables = [x.strip() for y in table_vals for x in y] - iterable = [l for x in algs_vals for l in list(x.keys())] + iterable = [alg for x in algs_vals for alg in list(x.keys())] iterable += tables all_algorithms = set() for x in iterable: - if '#' in x: + if "#" in x: # erase everything until "#" included and take digits - all_algorithms.add( - ''.join(filter(str.isdigit, x[x.index('#') + 1:]))) + all_algorithms.add("".join(filter(str.isdigit, x[x.index("#") + 1 :]))) else: - all_algorithms.add(''.join(filter(str.isdigit, x))) + all_algorithms.add("".join(filter(str.isdigit, x))) not_found = [] - + if cert.web_scan.algorithms is None: raise RuntimeError(f"Algorithms were not found for cert {cert.dgst} - this should not be happening.") - - for alg_list in [a['Certificate'] for a in cert.web_scan.algorithms]: + + for alg_list in [a["Certificate"] for a in cert.web_scan.algorithms]: for web_alg in alg_list: - if ''.join(filter(str.isdigit, web_alg)) not in all_algorithms: + if "".join(filter(str.isdigit, web_alg)) not in all_algorithms: not_found.append(web_alg) return len(not_found) @@ -548,13 +595,13 @@ class FIPSCertificate(Certificate, ComplexSerializableType): def remove_platforms(text_to_parse: str): pat = re.compile(r"(?:(?:modification|revision|change) history|version control)\n[\s\S]*? ", re.IGNORECASE) for match in pat.finditer(text_to_parse): - text_to_parse = text_to_parse.replace( - match.group(), 'x' * len(match.group())) + text_to_parse = text_to_parse.replace(match.group(), "x" * len(match.group())) return text_to_parse @staticmethod - def parse_cert_file_common(text_to_parse: str, whole_text_with_newlines: str, - search_rules: Dict) -> Tuple[Dict[Pattern, Dict], str]: + def parse_cert_file_common( + text_to_parse: str, whole_text_with_newlines: str, search_rules: Dict + ) -> Tuple[Dict[Pattern, Dict], str]: # apply all rules items_found_all: Dict[Pattern, Dict] = {} for rule_group in search_rules.keys(): @@ -583,15 +630,13 @@ class FIPSCertificate(Certificate, ComplexSerializableType): MAX_ALLOWED_MATCH_LENGTH = 300 match_len = len(match) if match_len > MAX_ALLOWED_MATCH_LENGTH: - logger.warning('Excessive match with length of {} detected for rule {}'.format( - match_len, rule)) + logger.warning("Excessive match with length of {} detected for rule {}".format(match_len, rule)) if match not in items_found[rule_str]: items_found[rule_str][match] = {} items_found[rule_str][match][constants.TAG_MATCH_COUNTER] = 0 - if sec_certs.constants.APPEND_DETAILED_MATCH_MATCHES: - items_found[rule_str][match][constants.TAG_MATCH_MATCHES] = [ - ] + if constants.APPEND_DETAILED_MATCH_MATCHES: + items_found[rule_str][match][constants.TAG_MATCH_MATCHES] = [] # else: # items_found[rule_str][match][TAG_MATCH_MATCHES] = ['List of matches positions disabled. Set APPEND_DETAILED_MATCH_MATCHES to True'] @@ -601,9 +646,8 @@ class FIPSCertificate(Certificate, ComplexSerializableType): # line_number = get_line_number(lines, line_length_compensation, match_span[0]) # start index, end index, line number # items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number]) - if sec_certs.constants.APPEND_DETAILED_MATCH_MATCHES: - items_found[rule_str][match][constants.TAG_MATCH_MATCHES].append( - [match_span[0], match_span[1]]) + if constants.APPEND_DETAILED_MATCH_MATCHES: + items_found[rule_str][match][constants.TAG_MATCH_MATCHES].append([match_span[0], match_span[1]]) # highlight all found strings (by xxxxx) from the input text and store the rest all_matches = [] @@ -617,8 +661,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): # sort before replacement based on the length of match all_matches.sort(key=len, reverse=True) for match in all_matches: - whole_text_with_newlines = whole_text_with_newlines.replace( - match, 'x' * len(match)) + whole_text_with_newlines = whole_text_with_newlines.replace(match, "x" * len(match)) return items_found_all, whole_text_with_newlines @@ -643,7 +686,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): match = m.group() match = normalize_match_string(match) - if match == '': + if match == "": continue if match not in items_found[rule.pattern]: @@ -652,33 +695,33 @@ class FIPSCertificate(Certificate, ComplexSerializableType): items_found[rule.pattern][match][constants.TAG_MATCH_COUNTER] += 1 - text_to_parse = text_to_parse.replace( - match, 'x' * len(match)) + text_to_parse = text_to_parse.replace(match, "x" * len(match)) return items_found_all, text_to_parse @staticmethod - def analyze_tables(tup: Tuple['FIPSCertificate', bool]) -> Tuple[bool, 'FIPSCertificate', List]: + def analyze_tables(tup: Tuple["FIPSCertificate", bool]) -> Tuple[bool, "FIPSCertificate", List]: cert, precision = tup - if (not precision and cert.state.tables_done) \ - or (precision and cert.heuristics.unmatched_algs < config.cert_threshold): + if (not precision and cert.state.tables_done) or ( + precision and cert.heuristics.unmatched_algs < config.cert_threshold + ): return cert.state.tables_done, cert, [] cert_file = cert.state.sp_path - txt_file = cert_file.with_suffix('.pdf.txt') - with open(txt_file, 'r', encoding='utf-8') as f: + txt_file = cert_file.with_suffix(".pdf.txt") + with open(txt_file, "r", encoding="utf-8") as f: tables = helpers.find_tables(f.read(), txt_file) all_pages = precision and cert.heuristics.unmatched_algs > config.cert_threshold # bool value lst: List = [] if tables: try: - data = read_pdf(cert_file, pages='all' if all_pages else tables, silent=True) + data = read_pdf(cert_file, pages="all" if all_pages else tables, silent=True) except Exception as e: try: logger.error(e) helpers.repair_pdf(cert_file) - data = read_pdf(cert_file, pages='all' if all_pages else tables, silent=True) + data = read_pdf(cert_file, pages="all" if all_pages else tables, silent=True) except Exception as ex: logger.error(ex) @@ -687,9 +730,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): # find columns with cert numbers for df in data: for col in range(len(df.columns)): - if 'cert' in df.columns[col].lower() or 'algo' in df.columns[col].lower(): - tmp = FIPSCertificate.extract_algorithm_certificates( - df.iloc[:, col].to_string(index=False), True) + if "cert" in df.columns[col].lower() or "algo" in df.columns[col].lower(): + tmp = FIPSCertificate.extract_algorithm_certificates( + df.iloc[:, col].to_string(index=False), True + ) lst += tmp if tmp != [{"Certificate": []}] else [] # Parse again if someone picks not so descriptive column names tmp = FIPSCertificate.extract_algorithm_certificates(df.to_string(index=False)) @@ -698,12 +742,12 @@ class FIPSCertificate(Certificate, ComplexSerializableType): def _create_alg_set(self) -> Set: result: Set[str] = set() - + if self.web_scan.algorithms is None: raise RuntimeError(f"Algorithms were not found for cert {self.dgst} - this should not be happening.") - + for alg in self.web_scan.algorithms: - result.update(cert for cert in alg['Certificate']) + result.update(cert for cert in alg["Certificate"]) return result def remove_algorithms(self): @@ -715,60 +759,62 @@ class FIPSCertificate(Certificate, ComplexSerializableType): # TODO figure out why can't I delete this if self.web_scan.mentioned_certs: for item, value in self.web_scan.mentioned_certs.items(): - self.heuristics.keywords['rules_cert_id'].update({'caveat_item': {item: value}}) + self.heuristics.keywords["rules_cert_id"].update({"caveat_item": {item: value}}) alg_set = self._create_alg_set() - for rule in self.heuristics.keywords['rules_cert_id']: + for rule in self.heuristics.keywords["rules_cert_id"]: to_pop = set() rr = re.compile(rule) - for cert in self.heuristics.keywords['rules_cert_id'][rule]: + for cert in self.heuristics.keywords["rules_cert_id"][rule]: if cert in alg_set: to_pop.add(cert) continue - for alg in self.heuristics.keywords['rules_fips_algorithms']: - for found in self.heuristics.keywords['rules_fips_algorithms'][alg]: - if rr.search(found) \ - and rr.search(cert) \ - and rr.search(found).group('id') == rr.search(cert).group('id'): + for alg in self.heuristics.keywords["rules_fips_algorithms"]: + for found in self.heuristics.keywords["rules_fips_algorithms"][alg]: + if ( + rr.search(found) + and rr.search(cert) + and rr.search(found).group("id") == rr.search(cert).group("id") + ): to_pop.add(cert) for alg_cert in self.heuristics.algorithms: - for cert_no in alg_cert['Certificate']: - if int(''.join(filter(str.isdigit, cert_no))) == int(''.join(filter(str.isdigit, cert))): + for cert_no in alg_cert["Certificate"]: + if int("".join(filter(str.isdigit, cert_no))) == int("".join(filter(str.isdigit, cert))): to_pop.add(cert) for r in to_pop: - self.heuristics.keywords['rules_cert_id'][rule].pop(r, None) + self.heuristics.keywords["rules_cert_id"][rule].pop(r, None) - self.heuristics.keywords['rules_cert_id'][rule].pop( - self.cert_id, None) + self.heuristics.keywords["rules_cert_id"][rule].pop(self.cert_id, None) @staticmethod def get_compare(vendor: str): - vendor_split = vendor.replace(',', '') \ - .replace('-', ' ').replace('+', ' ').replace('®', '').replace('(R)', '').split() + vendor_split = ( + vendor.replace(",", "").replace("-", " ").replace("+", " ").replace("®", "").replace("(R)", "").split() + ) return vendor_split[0][:4] if len(vendor_split) > 0 else vendor def compute_heuristics_version(self): - versions_for_extraction = '' + versions_for_extraction = "" if self.web_scan.module_name: - versions_for_extraction += f' {self.web_scan.module_name}' + versions_for_extraction += f" {self.web_scan.module_name}" if self.web_scan.hw_version: - versions_for_extraction += f' {self.web_scan.hw_version}' + versions_for_extraction += f" {self.web_scan.hw_version}" if self.web_scan.fw_version: - versions_for_extraction += f' {self.web_scan.fw_version}' + versions_for_extraction += f" {self.web_scan.fw_version}" self.heuristics.extracted_versions = helpers.compute_heuristics_version(versions_for_extraction) # TODO: This function is probably safe to delete // I'll not type it then - older API probably? def compute_heuristics_cpe_vendors(self, cpe_dataset: CPEDataset): if self.web_scan.vendor is None: raise RuntimeError(f"Vendor for cert {self.dgst} not found - this should not be happening.") - self.heuristics.cpe_candidate_vendors = cpe_dataset.get_candidate_list_of_vendors(self.web_scan.vendor) # type: ignore + self.heuristics.cpe_candidate_vendors = cpe_dataset.get_candidate_list_of_vendors(self.web_scan.vendor) # type: ignore def compute_heuristics_cpe_match(self, cpe_classifier: CPEClassifier): if not self.web_scan.module_name: self.heuristics.cpe_matches = None else: - self.heuristics.cpe_matches = cpe_classifier.predict_single_cert(self.web_scan.vendor, # type: ignore - self.web_scan.module_name, - self.heuristics.extracted_versions) + self.heuristics.cpe_matches = cpe_classifier.predict_single_cert( + self.web_scan.vendor, self.web_scan.module_name, self.heuristics.extracted_versions # type: ignore + ) diff --git a/sec_certs/sample/protection_profile.py b/sec_certs/sample/protection_profile.py index bb7cc788..d8825ced 100644 --- a/sec_certs/sample/protection_profile.py +++ b/sec_certs/sample/protection_profile.py @@ -1,11 +1,10 @@ -from dataclasses import dataclass -from typing import Optional, FrozenSet import copy -from typing import Dict import logging +from dataclasses import dataclass +from typing import Dict, FrozenSet, Optional -from sec_certs.serialization.json import ComplexSerializableType import sec_certs.helpers as helpers +from sec_certs.serialization.json import ComplexSerializableType logger = logging.getLogger(__name__) @@ -15,25 +14,26 @@ class ProtectionProfile(ComplexSerializableType): """ Object for holding protection profiles. """ - pp_name: Optional[str] + + pp_name: str pp_link: Optional[str] = None pp_ids: Optional[FrozenSet[str]] = None def __post_init__(self): - super().__setattr__('pp_name', helpers.sanitize_string(self.pp_name)) - super().__setattr__('pp_link', helpers.sanitize_link(self.pp_link)) + super().__setattr__("pp_name", helpers.sanitize_string(self.pp_name)) + super().__setattr__("pp_link", helpers.sanitize_link(self.pp_link)) @classmethod def from_dict(cls, dct): new_dct = copy.deepcopy(dct) - new_dct['pp_ids'] = frozenset(new_dct['pp_ids']) if new_dct['pp_ids'] else None + new_dct["pp_ids"] = frozenset(new_dct["pp_ids"]) if new_dct["pp_ids"] else None return cls(*tuple(new_dct.values())) @classmethod def from_old_api_dict(cls, dct: Dict): - pp_name = helpers.sanitize_string(dct['csv_scan']['cc_pp_name']) - pp_link = helpers.sanitize_link(dct['csv_scan']['link_pp_document']) - pp_ids = frozenset(dct['processed']['cc_pp_csvid']) if dct['processed']['cc_pp_csvid'] else None + pp_name = helpers.sanitize_string(dct["csv_scan"]["cc_pp_name"]) + pp_link = helpers.sanitize_link(dct["csv_scan"]["link_pp_document"]) + pp_ids = frozenset(dct["processed"]["cc_pp_csvid"]) if dct["processed"]["cc_pp_csvid"] else None return cls(pp_name, pp_link, pp_ids) def __eq__(self, other): diff --git a/sec_certs/serialization/json.py b/sec_certs/serialization/json.py index dce3ac6b..8c229f65 100644 --- a/sec_certs/serialization/json.py +++ b/sec_certs/serialization/json.py @@ -1,8 +1,8 @@ +import copy import json from datetime import date from pathlib import Path -from typing import Dict, List, Union, Optional, Callable -import copy +from typing import Callable, Dict, List, Optional, Union class ComplexSerializableType: @@ -20,22 +20,26 @@ class ComplexSerializableType: try: return cls(*(tuple(dct.values()))) except TypeError as e: - raise TypeError(f'Dict: {dct} on {cls.__mro__}') from e + raise TypeError(f"Dict: {dct} on {cls.__mro__}") from e def to_json(self, output_path: Optional[Union[str, Path]] = None): - if output_path is None and not hasattr(self, 'json_path'): - raise ValueError(f'The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument.') + if output_path is None and not hasattr(self, "json_path"): + raise ValueError( + f"The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument." + ) elif output_path is None and self.json_path is None: # type: ignore - raise ValueError(f'The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument.') + raise ValueError( + f"The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument." + ) elif output_path is None: output_path = self.json_path # type: ignore - with Path(output_path).open('w') as handle: + with Path(output_path).open("w") as handle: json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False) @classmethod def from_json(cls, input_path: Union[str, Path]): input_path = Path(input_path) - with input_path.open('r') as handle: + with input_path.open("r") as handle: obj = json.load(handle, cls=CustomJSONDecoder) return obj @@ -44,20 +48,23 @@ class ComplexSerializableType: def serialize(func: Callable): def inner_func(*args, **kwargs): if not args or not issubclass(type(args[0]), ComplexSerializableType): - raise ValueError('@serialize decorator is to be used only on instance methods of ComplexSerializableType child classes.') + raise ValueError( + "@serialize decorator is to be used only on instance methods of ComplexSerializableType child classes." + ) - update_json = kwargs.pop('update_json', True) + update_json = kwargs.pop("update_json", True) result = func(*args, **kwargs) if update_json: args[0].to_json() return result + return inner_func class CustomJSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, ComplexSerializableType): - return {**{'_type': type(obj).__name__}, **obj.to_dict()} + return {**{"_type": type(obj).__name__}, **obj.to_dict()} if isinstance(obj, dict): return obj if isinstance(obj, set): @@ -77,13 +84,14 @@ class CustomJSONDecoder(json.JSONDecoder): ComplexSerializableType (nested inheritance does not currently work (because x.__subclassess__() prints only direct subclasses. Any such class must implement methods to_dict() and from_dict(). These are used to drive serialization. """ + def __init__(self, *args, **kwargs): json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) self.serializable_complex_types = {x.__name__: x for x in ComplexSerializableType.__subclasses__()} def object_hook(self, obj): - if '_type' in obj and obj['_type'] in self.serializable_complex_types.keys(): - complex_type = obj.pop('_type') + if "_type" in obj and obj["_type"] in self.serializable_complex_types.keys(): + complex_type = obj.pop("_type") return self.serializable_complex_types[complex_type].from_dict(obj) return obj diff --git a/sec_certs/serialization/pandas.py b/sec_certs/serialization/pandas.py index e7de4d67..dc0aae18 100644 --- a/sec_certs/serialization/pandas.py +++ b/sec_certs/serialization/pandas.py @@ -8,9 +8,9 @@ class PandasSerializableType(ABC): @property @abstractmethod def pandas_tuple(self): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") @property @abstractmethod def pandas_columns(self): - raise NotImplementedError('Not meant to be implemented') + raise NotImplementedError("Not meant to be implemented") @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -from setuptools import setup, find_packages +from setuptools import find_packages, setup -with open('requirements.txt') as f: +with open("requirements.txt") as f: requirements = f.read().splitlines() setup( - name='sec-certs', - author='Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky', - author_email='svenda@fi.muni.cz', + name="sec-certs", + author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky", + author_email="svenda@fi.muni.cz", version_config=True, - setup_requires=['setuptools-git-versioning'], + setup_requires=["setuptools-git-versioning"], packages=find_packages(), - license='MIT', + license="MIT", description="Tool for analysis of security certificates", long_description=open("README.md").read(), long_description_content_type="text/markdown", @@ -24,22 +24,12 @@ setup( "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Intended Audience :: Developers", - "Intended Audience :: Science/Research" + "Intended Audience :: Science/Research", ], python_requires=">=3.8", install_requires=requirements, - extras_require={ - "dev": ["mypy", "flake8"], - "test": ["pytest", "coverage"] - }, + extras_require={"dev": ["mypy", "flake8"], "test": ["pytest", "coverage"]}, include_package_data=True, - package_data={ - 'sec_certs': ['settings.yaml', 'settings-schema.json'] - }, - entry_points={ - 'console_scripts': [ - 'cc-certs=cc_cli:main', - 'fips-certs=fips_cli:main' - ] - } + package_data={"sec_certs": ["settings.yaml", "settings-schema.json"]}, + entry_points={"console_scripts": ["cc-certs=cc_cli:main", "fips-certs=fips_cli:main"]}, ) diff --git a/tests/fips_test_utils.py b/tests/fips_test_utils.py index 94fb406a..2fce325b 100644 --- a/tests/fips_test_utils.py +++ b/tests/fips_test_utils.py @@ -1,17 +1,18 @@ -from typing import List from pathlib import Path +from typing import List + def generate_html(ids: List[str], path: Path): def generate_entry(certificate_id: str) -> str: - return f''' + return f""" <tr id="cert-row-0"> <td class="text-center"> <a href="/projects/cryptographic-module-validation-program/certificate/3898" id="cert-number-link-0">{certificate_id}</a> </td> </tr> - ''' + """ - html_head = ''' + html_head = """ <!DOCTYPE html> <html lang="en-us" xml:lang="en-us"> <head> @@ -25,11 +26,11 @@ def generate_html(ids: List[str], path: Path): <meta name="theme-color" content="#000000" /> <meta name="google-site-verification" content="xbrnrVYDgLD-Bd64xHLCt4XsPXzUhQ-4lGMj4TdUUTA" /> </head> - ''' + """ rows = "" for cert_id in ids: rows += f"\n{generate_entry(cert_id)}\n" - html_body = f''' + html_body = f""" <body> <table class="table table-striped table-condensed publications-table table-bordered" id="searchResultsTable"> <thead> @@ -46,6 +47,6 @@ def generate_html(ids: List[str], path: Path): </tbody> </table> </body> - ''' - with open(path, 'w') as f: + """ + with open(path, "w") as f: f.write(f"{html_head}\n{html_body}\n") diff --git a/tests/test_cc_heuristics.py b/tests/test_cc_heuristics.py index 13cdb465..caa6b1fb 100644 --- a/tests/test_cc_heuristics.py +++ b/tests/test_cc_heuristics.py @@ -1,64 +1,72 @@ -import copy -import datetime +import shutil import tempfile +from pathlib import Path +from typing import ClassVar, Dict, List from unittest import TestCase + +import tests.data.test_cc_heuristics from sec_certs.dataset.common_criteria import CCDataset -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.protection_profile import ProtectionProfile from sec_certs.dataset.cpe import CPEDataset -from sec_certs.sample.cpe import CPE from sec_certs.dataset.cve import CVEDataset +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.cpe import CPE from sec_certs.sample.cve import CVE -from pathlib import Path -from typing import ClassVar, Dict -import shutil - -import tests.data.test_cc_heuristics +from sec_certs.sample.protection_profile import ProtectionProfile class TestCommonCriteriaHeuristics(TestCase): - dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / 'vulnerable_dataset.json' + dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / "vulnerable_dataset.json" # type: ignore # mypy issue #1422 data_dir_path: ClassVar[Path] = dataset_json_path.parent + tmp_dir: ClassVar[tempfile.TemporaryDirectory] + cc_dset: CCDataset + cve_dset: CVEDataset + cves: List[CVE] + cpe_dset: CPEDataset + cpes: List[CPE] @classmethod def setUpClass(cls) -> None: - cls.tmp_dir: ClassVar[tempfile.TemporaryDirectory] = tempfile.TemporaryDirectory() + cls.tmp_dir = tempfile.TemporaryDirectory() shutil.copytree(cls.data_dir_path, cls.tmp_dir.name, dirs_exist_ok=True) - cls.cc_dset: CCDataset = CCDataset.from_json(Path(cls.tmp_dir.name) / 'vulnerable_dataset.json') + cls.cc_dset = CCDataset.from_json(Path(cls.tmp_dir.name) / "vulnerable_dataset.json") cls.cc_dset.process_protection_profiles() cls.cc_dset.download_all_pdfs() cls.cc_dset.convert_all_pdfs() cls.cc_dset._extract_data() cls.cc_dset._compute_heuristics(use_nist_cpe_matching_dict=False) - - cpe_single_sign_on = CPE("cpe:2.3:a:ibm:security_access_manager_for_enterprise_single_sign-on:8.2.2:*:*:*:*:*:*:*", - "IBM Security Access Manager For Enterprise Single Sign-On 8.2.2") + cpe_single_sign_on = CPE( + "cpe:2.3:a:ibm:security_access_manager_for_enterprise_single_sign-on:8.2.2:*:*:*:*:*:*:*", + "IBM Security Access Manager For Enterprise Single Sign-On 8.2.2", + ) cls.cpes = [ cpe_single_sign_on, - CPE("cpe:2.3:a:ibm:security_key_lifecycle_manager:2.6.0.1:*:*:*:*:*:*:*", - "IBM Security Key Lifecycle Manager 2.6.0.1"), - CPE("cpe:2.3:a:semperplugins:all_in_one_seo_pack:1.3.6.4:*:*:*:*:wordpress:*:*", - "Semper Plugins All in One SEO Pack 1.3.6.4 for WordPress"), - CPE("cpe:2.3:a:tracker-software:pdf-xchange_lite_printer:6.0.320.0:*:*:*:*:*:*:*", - "Tracker Software PDF-XChange Lite Printer 6.0.320.0") + CPE( + "cpe:2.3:a:ibm:security_key_lifecycle_manager:2.6.0.1:*:*:*:*:*:*:*", + "IBM Security Key Lifecycle Manager 2.6.0.1", + ), + CPE( + "cpe:2.3:a:semperplugins:all_in_one_seo_pack:1.3.6.4:*:*:*:*:wordpress:*:*", + "Semper Plugins All in One SEO Pack 1.3.6.4 for WordPress", + ), + CPE( + "cpe:2.3:a:tracker-software:pdf-xchange_lite_printer:6.0.320.0:*:*:*:*:*:*:*", + "Tracker Software PDF-XChange Lite Printer 6.0.320.0", + ), ] - cls.cpe_dset = CPEDataset(True, Path('../'), {x.uri: x for x in cls.cpes}) + cls.cpe_dset = CPEDataset(True, Path("../"), {x.uri: x for x in cls.cpes}) cls.cves = [ - CVE('CVE-2017-1732', - [cpe_single_sign_on], - CVE.Impact(5.3, 'MEDIUM', 3.9, 1.4), - '2021-05-26T04:15Z', - ), - CVE('CVE-2019-4513', + CVE( + "CVE-2017-1732", [cpe_single_sign_on], - CVE.Impact(8.2, 'HIGH', 3.9, 4.2), - '2000-05-26T04:15Z' - ) - ] + CVE.Impact(5.3, "MEDIUM", 3.9, 1.4), + "2021-05-26T04:15Z", + ), + CVE("CVE-2019-4513", [cpe_single_sign_on], CVE.Impact(8.2, "HIGH", 3.9, 4.2), "2000-05-26T04:15Z"), + ] cls.cve_dset = CVEDataset({x.cve_id: x for x in cls.cves}) cls.cve_dset.build_lookup_dict(use_nist_mapping=False) @@ -67,89 +75,172 @@ class TestCommonCriteriaHeuristics(TestCase): cls.tmp_dir.cleanup() def test_load_cpe_dataset(self): - json_cpe_dset = CPEDataset.from_json(self.data_dir_path / 'auxillary_datasets' / 'cpe_dataset.json') - json_cpe_dset.json_path = Path('../') - self.assertEqual(self.cpe_dset, json_cpe_dset, 'CPE template dataset does not match CPE dataset loaded from json.') + json_cpe_dset = CPEDataset.from_json(self.data_dir_path / "auxillary_datasets" / "cpe_dataset.json") + json_cpe_dset.json_path = Path("../") + self.assertEqual( + self.cpe_dset, json_cpe_dset, "CPE template dataset does not match CPE dataset loaded from json." + ) def test_cpe_lookup_dicts(self): - self.assertEqual(self.cpe_dset.vendors, {'ibm', 'tracker-software', 'semperplugins'}, - 'The set of versions in CPE dataset does not match template') - self.assertEqual(self.cpe_dset.vendor_to_versions, {'ibm': {'8.2.2', '2.6.0.1'}, 'semperplugins': {'1.3.6.4'}, 'tracker-software': {'6.0.320.0'}}, - 'The CPE lookup dictionary vendor->version of CPE dataset does not match template.') - self.assertEqual(self.cpe_dset.vendor_version_to_cpe, {('ibm', '8.2.2'): {CPE('cpe:2.3:a:ibm:security_access_manager_for_enterprise_single_sign-on:8.2.2:*:*:*:*:*:*:*', 'IBM Security Access Manager For Enterprise Single Sign-On 8.2.2')}, ('ibm', '2.6.0.1'): {CPE('cpe:2.3:a:ibm:security_key_lifecycle_manager:2.6.0.1:*:*:*:*:*:*:*', 'IBM Security Key Lifecycle Manager 2.6.0.1')}, ('semperplugins', '1.3.6.4'): {CPE('cpe:2.3:a:semperplugins:all_in_one_seo_pack:1.3.6.4:*:*:*:*:wordpress:*:*', 'Semper Plugins All in One SEO Pack 1.3.6.4 for WordPress')}, ('tracker-software', '6.0.320.0'): {CPE('cpe:2.3:a:tracker-software:pdf-xchange_lite_printer:6.0.320.0:*:*:*:*:*:*:*', 'Tracker Software PDF-XChange Lite Printer 6.0.320.0')}}, - 'The CPE lookup dictionary (vendor,version)->cpe does not match the template.') + self.assertEqual( + self.cpe_dset.vendors, + {"ibm", "tracker-software", "semperplugins"}, + "The set of versions in CPE dataset does not match template", + ) + self.assertEqual( + self.cpe_dset.vendor_to_versions, + {"ibm": {"8.2.2", "2.6.0.1"}, "semperplugins": {"1.3.6.4"}, "tracker-software": {"6.0.320.0"}}, + "The CPE lookup dictionary vendor->version of CPE dataset does not match template.", + ) + self.assertEqual( + self.cpe_dset.vendor_version_to_cpe, + { + ("ibm", "8.2.2"): { + CPE( + "cpe:2.3:a:ibm:security_access_manager_for_enterprise_single_sign-on:8.2.2:*:*:*:*:*:*:*", + "IBM Security Access Manager For Enterprise Single Sign-On 8.2.2", + ) + }, + ("ibm", "2.6.0.1"): { + CPE( + "cpe:2.3:a:ibm:security_key_lifecycle_manager:2.6.0.1:*:*:*:*:*:*:*", + "IBM Security Key Lifecycle Manager 2.6.0.1", + ) + }, + ("semperplugins", "1.3.6.4"): { + CPE( + "cpe:2.3:a:semperplugins:all_in_one_seo_pack:1.3.6.4:*:*:*:*:wordpress:*:*", + "Semper Plugins All in One SEO Pack 1.3.6.4 for WordPress", + ) + }, + ("tracker-software", "6.0.320.0"): { + CPE( + "cpe:2.3:a:tracker-software:pdf-xchange_lite_printer:6.0.320.0:*:*:*:*:*:*:*", + "Tracker Software PDF-XChange Lite Printer 6.0.320.0", + ) + }, + }, + "The CPE lookup dictionary (vendor,version)->cpe does not match the template.", + ) def test_cve_lookup_dicts(self): alt_lookup = {x: set(y) for x, y in self.cve_dset.cpe_to_cve_ids_lookup.items()} - self.assertEqual(alt_lookup, {'cpe:2.3:a:ibm:security_access_manager_for_enterprise_single_sign-on:8.2.2:*:*:*:*:*:*:*': {x.cve_id for x in self.cves}}, - 'The CVE lookup dicionary cve-> affected cpes does not match the template') + self.assertEqual( + alt_lookup, + { + "cpe:2.3:a:ibm:security_access_manager_for_enterprise_single_sign-on:8.2.2:*:*:*:*:*:*:*": { + x.cve_id for x in self.cves + } + }, + "The CVE lookup dicionary cve-> affected cpes does not match the template", + ) def test_load_cve_dataset(self): - json_cve_dset = CVEDataset.from_json(self.data_dir_path / 'auxillary_datasets' / 'cve_dataset.json') - self.assertEqual(self.cve_dset, json_cve_dset, 'CVE template dataset does not match CVE dataset loaded from json.') + json_cve_dset = CVEDataset.from_json(self.data_dir_path / "auxillary_datasets" / "cve_dataset.json") + self.assertEqual( + self.cve_dset, json_cve_dset, "CVE template dataset does not match CVE dataset loaded from json." + ) def test_match_cpe(self): - self.assertTrue(self.cpes[0].uri in self.cc_dset['ebd276cca70fd723'].heuristics.cpe_matches, 'The CPE matching algorithm did not find the right CPE.') - self.assertTrue(len(self.cc_dset['ebd276cca70fd723'].heuristics.cpe_matches) == 1, 'Exactly one CPE match should be found.') + self.assertTrue( + self.cpes[0].uri in self.cc_dset["ebd276cca70fd723"].heuristics.cpe_matches, + "The CPE matching algorithm did not find the right CPE.", + ) + self.assertTrue( + len(self.cc_dset["ebd276cca70fd723"].heuristics.cpe_matches) == 1, "Exactly one CPE match should be found." + ) def test_find_related_cves(self): - self.cc_dset['ebd276cca70fd723'].heuristics.cpe_matches = [self.cpes[0].uri] - self.cc_dset.compute_related_cves(use_nist_cpe_matching_dict = False) - self.assertEqual({x.cve_id for x in self.cves}, self.cc_dset['ebd276cca70fd723'].heuristics.related_cves, 'The computed CVEs do not match the excpected CVEs') + self.cc_dset["ebd276cca70fd723"].heuristics.cpe_matches = [self.cpes[0].uri] + self.cc_dset.compute_related_cves(use_nist_cpe_matching_dict=False) + self.assertEqual( + {x.cve_id for x in self.cves}, + self.cc_dset["ebd276cca70fd723"].heuristics.related_cves, + "The computed CVEs do not match the excpected CVEs", + ) def test_version_extraction(self): - self.assertEqual(self.cc_dset['ebd276cca70fd723'].heuristics.extracted_versions, ['8.2'], 'The version extracted from the sample does not match the template') - new_cert = CommonCriteriaCert('', '', 'IDOneClassIC Card : ID-One Cosmo 64 RSA v5.4 and applet IDOneClassIC v1.0 embedded on P5CT072VOP', '', '', - '', None, None, '', '', '', '', set(), set(), None, None, None) + self.assertEqual( + self.cc_dset["ebd276cca70fd723"].heuristics.extracted_versions, + ["8.2"], + "The version extracted from the sample does not match the template", + ) + new_cert = CommonCriteriaCert( + "", + "", + "IDOneClassIC Card : ID-One Cosmo 64 RSA v5.4 and applet IDOneClassIC v1.0 embedded on P5CT072VOP", + "", + "", + "", + None, + None, + "", + "", + "", + "", + set(), + set(), + None, + None, + None, + ) new_cert.compute_heuristics_version() - self.assertEqual(set(new_cert.heuristics.extracted_versions), {'5.4', '1.0'}, 'The extracted versions do not match the template.') + self.assertEqual( + set(new_cert.heuristics.extracted_versions), + {"5.4", "1.0"}, + "The extracted versions do not match the template.", + ) def test_cert_lab_heuristics(self): - self.assertEqual(self.cc_dset['ebd276cca70fd723'].heuristics.cert_lab, ['BSI']) + self.assertEqual(self.cc_dset["ebd276cca70fd723"].heuristics.cert_lab, ["BSI"]) def test_cert_id_heuristics(self): - self.assertEqual(self.cc_dset['ebd276cca70fd723'].heuristics.cert_id, 'BSI-DSZ-CC-0683-2014') + self.assertEqual(self.cc_dset["ebd276cca70fd723"].heuristics.cert_id, "BSI-DSZ-CC-0683-2014") def test_keywords_heuristics(self): - extracted_keywords: Dict = self.cc_dset['ebd276cca70fd723'].pdf_data.st_keywords + extracted_keywords: Dict = self.cc_dset["ebd276cca70fd723"].pdf_data.st_keywords - self.assertTrue('rules_security_level' in extracted_keywords) - self.assertEqual(extracted_keywords['rules_security_level']['EAL3'], 1) + self.assertTrue("rules_security_level" in extracted_keywords) + self.assertEqual(extracted_keywords["rules_security_level"]["EAL3"], 1) - self.assertTrue('rules_security_assurance_components' in extracted_keywords) - self.assertEqual(extracted_keywords['rules_security_assurance_components']['ADV_ARC.1'], 1) - self.assertEqual(extracted_keywords['rules_security_assurance_components']['ADV_FSP.3'], 1) - self.assertEqual(extracted_keywords['rules_security_assurance_components']['ADV_TDS.2'], 1) + self.assertTrue("rules_security_assurance_components" in extracted_keywords) + self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_ARC.1"], 1) + self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_FSP.3"], 1) + self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_TDS.2"], 1) - self.assertTrue('rules_crypto_algs' in extracted_keywords) - self.assertEqual(extracted_keywords['rules_crypto_algs']['AES'], 2) + self.assertTrue("rules_crypto_algs" in extracted_keywords) + self.assertEqual(extracted_keywords["rules_crypto_algs"]["AES"], 2) - self.assertTrue('rules_block_cipher_modes' in extracted_keywords) - self.assertEqual(extracted_keywords['rules_block_cipher_modes']['CBC'], 2) + self.assertTrue("rules_block_cipher_modes" in extracted_keywords) + self.assertEqual(extracted_keywords["rules_block_cipher_modes"]["CBC"], 2) def test_protection_profiles_matching(self): - artificial_pp: ProtectionProfile = ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0', - 'http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf') - self.cc_dset['ebd276cca70fd723'].protection_profiles = {artificial_pp} - expected_pp: ProtectionProfile = ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0', - 'http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf', - frozenset(['KECS-PP-0822-2017 SSO V1.0'])) + artificial_pp: ProtectionProfile = ProtectionProfile( + "Korean National Protection Profile for Single Sign On V1.0", + "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf", + ) + self.cc_dset["ebd276cca70fd723"].protection_profiles = {artificial_pp} + expected_pp: ProtectionProfile = ProtectionProfile( + "Korean National Protection Profile for Single Sign On V1.0", + "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf", + frozenset(["KECS-PP-0822-2017 SSO V1.0"]), + ) self.cc_dset.process_protection_profiles(to_download=False) - self.assertSetEqual(self.cc_dset['ebd276cca70fd723'].protection_profiles, {expected_pp}) + self.assertSetEqual(self.cc_dset["ebd276cca70fd723"].protection_profiles, {expected_pp}) def test_single_record_dependency_heuristics(self): # Single record in daset is not affecting nor affected by other records - heuristics = self.cc_dset['ebd276cca70fd723'].heuristics + heuristics = self.cc_dset["ebd276cca70fd723"].heuristics self.assertEqual(heuristics.directly_affected_by, None) self.assertEqual(heuristics.indirectly_affected_by, None) self.assertEqual(heuristics.directly_affecting, None) self.assertEqual(heuristics.indirectly_affecting, None) def test_dependency_dataset(self): - dependency_dataset = CCDataset.from_json(self.data_dir_path / 'dependency_dataset.json') + dependency_dataset = CCDataset.from_json(self.data_dir_path / "dependency_dataset.json") dependency_dataset._compute_dependencies() test_cert: CommonCriteriaCert = dependency_dataset["692e91451741ef49"] - + self.assertEqual(test_cert.heuristics.directly_affected_by, ["BSI-DSZ-CC-0370-2006"]) self.assertEqual(test_cert.heuristics.indirectly_affected_by, {"BSI-DSZ-CC-0370-2006", "BSI-DSZ-CC-0517-2009"}) self.assertEqual(test_cert.heuristics.directly_affecting, {"BSI-DSZ-CC-0268-2005"}) diff --git a/tests/test_cc_oop.py b/tests/test_cc_oop.py index 3ca1935e..2426476d 100644 --- a/tests/test_cc_oop.py +++ b/tests/test_cc_oop.py @@ -1,178 +1,226 @@ -import tempfile -from unittest import TestCase -from pathlib import Path -from tempfile import TemporaryDirectory, mkstemp, NamedTemporaryFile -from datetime import date, datetime -import json import filecmp -import shutil import os +import shutil +import tempfile +from datetime import date, datetime +from pathlib import Path +from tempfile import NamedTemporaryFile, TemporaryDirectory +from unittest import TestCase +import sec_certs.constants as constants +import sec_certs.helpers as helpers from sec_certs.dataset.common_criteria import CCDataset from sec_certs.sample.common_criteria import CommonCriteriaCert from sec_certs.sample.protection_profile import ProtectionProfile -import sec_certs.helpers as helpers -import sec_certs.constants as constants class TestCommonCriteriaOOP(TestCase): def setUp(self): - self.test_data_dir = Path(__file__).parent / 'data' / 'test_cc_oop' - self.crt_one = CommonCriteriaCert('active', - 'Access Control Devices and Systems', - 'NetIQ Identity Manager 4.7', - 'NetIQ Corporation', - 'SE', - {'ALC_FLR.2', - 'EAL3+'}, - date(2020, 6, 15), - date(2025, 6, 15), - 'https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf', - 'https://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf', - 'https://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf', - 'https://www.netiq.com/', - set(), - set(), - None, - None, - None) + self.test_data_dir = Path(__file__).parent / "data" / "test_cc_oop" + self.crt_one = CommonCriteriaCert( + "active", + "Access Control Devices and Systems", + "NetIQ Identity Manager 4.7", + "NetIQ Corporation", + "SE", + {"ALC_FLR.2", "EAL3+"}, + date(2020, 6, 15), + date(2025, 6, 15), + "https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf", + "https://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf", + "https://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf", + "https://www.netiq.com/", + set(), + set(), + None, + None, + None, + ) - self.crt_two = CommonCriteriaCert('active', - 'Access Control Devices and Systems', - 'Magic SSO V4.0', - 'Dreamsecurity Co., Ltd.', - 'KR', - set(), - date(2019, 11, 15), - date(2024, 11, 15), - 'https://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf', - 'https://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf', - None, - 'https://www.dreamsecurity.com/', - {ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0', - 'https://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf')}, - set(), - None, - None, - None) + self.crt_two = CommonCriteriaCert( + "active", + "Access Control Devices and Systems", + "Magic SSO V4.0", + "Dreamsecurity Co., Ltd.", + "KR", + set(), + date(2019, 11, 15), + date(2024, 11, 15), + "https://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf", + "https://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf", + None, + "https://www.dreamsecurity.com/", + { + ProtectionProfile( + "Korean National Protection Profile for Single Sign On V1.0", + "https://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf", + ) + }, + set(), + None, + None, + None, + ) - pp = ProtectionProfile('sample_pp', 'https://sample.pp') - update = CommonCriteriaCert.MaintenanceReport(date(1900, 1, 1), 'Sample maintenance', 'https://maintenance.up', 'https://maintenance.up') - self.fictional_cert = CommonCriteriaCert('archived', - 'Sample category', - 'Sample certificate name', - 'Sample manufacturer', - 'Sample scheme', - {'Sample security level'}, - date(1900, 1, 2), - date(1900, 1, 3), - 'https://path.to/report/link', - 'https://path.to/st/link', - 'https://path.to/cert/link', - 'https://path.to/manufacturer/web', - {pp}, - {update}, - None, - None, - None) - self.template_dataset = CCDataset({self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, Path('/fictional/path/to/dataset'), 'toy dataset', 'toy dataset description') + pp = ProtectionProfile("sample_pp", "https://sample.pp") + update = CommonCriteriaCert.MaintenanceReport( + date(1900, 1, 1), "Sample maintenance", "https://maintenance.up", "https://maintenance.up" + ) + self.fictional_cert = CommonCriteriaCert( + "archived", + "Sample category", + "Sample certificate name", + "Sample manufacturer", + "Sample scheme", + {"Sample security level"}, + date(1900, 1, 2), + date(1900, 1, 3), + "https://path.to/report/link", + "https://path.to/st/link", + "https://path.to/cert/link", + "https://path.to/manufacturer/web", + {pp}, + {update}, + None, + None, + None, + ) + self.template_dataset = CCDataset( + {self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, + Path("/fictional/path/to/dataset"), + "toy dataset", + "toy dataset description", + ) self.template_dataset.timestamp = datetime(2020, 11, 16, hour=17, minute=4, second=14, microsecond=770153) self.template_dataset.state.meta_sources_parsed = True - self.template_report_pdf_hashes = {'309ac2fd7f2dcf17': '774c41fbba980191ca40ae610b2f61484c5997417b3325b6fd68b345173bde52', - '8cf86948f02f047d': '533a5995ef8b736cc48cfda30e8aafec77d285511471e0e5a9e8007c8750203a'} - self.template_target_pdf_hashes = {'309ac2fd7f2dcf17': 'b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9', - '8cf86948f02f047d': '3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be'} + self.template_report_pdf_hashes = { + "309ac2fd7f2dcf17": "774c41fbba980191ca40ae610b2f61484c5997417b3325b6fd68b345173bde52", + "8cf86948f02f047d": "533a5995ef8b736cc48cfda30e8aafec77d285511471e0e5a9e8007c8750203a", + } + self.template_target_pdf_hashes = { + "309ac2fd7f2dcf17": "b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9", + "8cf86948f02f047d": "3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be", + } - self.template_report_txt_path = self.test_data_dir / 'report_869415cc4b91282e.txt' - self.template_target_txt_path = self.test_data_dir / 'target_869415cc4b91282e.txt' + self.template_report_txt_path = self.test_data_dir / "report_869415cc4b91282e.txt" + self.template_target_txt_path = self.test_data_dir / "target_869415cc4b91282e.txt" def test_certificate_input_sanity(self): - self.assertEqual(self.crt_one.report_link, - 'https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf', - 'Report link contains some improperly escaped characters.') + self.assertEqual( + self.crt_one.report_link, + "https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf", + "Report link contains some improperly escaped characters.", + ) def test_download_and_convert_pdfs(self): - dset = CCDataset.from_json(self.test_data_dir / 'toy_dataset.json') + dset = CCDataset.from_json(self.test_data_dir / "toy_dataset.json") with TemporaryDirectory() as td: dset.root_dir = Path(td) dset.download_all_pdfs() dset.convert_all_pdfs() - actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.report_pdf_path) for key, val in dset.certs.items()} - actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.st_pdf_path) for key, val in dset.certs.items()} + actual_report_pdf_hashes = { + key: helpers.get_sha256_filepath(val.state.report_pdf_path) for key, val in dset.certs.items() + } + actual_target_pdf_hashes = { + key: helpers.get_sha256_filepath(val.state.st_pdf_path) for key, val in dset.certs.items() + } - self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (sample report) do not the template') - self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template') + self.assertEqual( + actual_report_pdf_hashes, + self.template_report_pdf_hashes, + "Hashes of downloaded pdfs (sample report) do not the template", + ) + self.assertEqual( + actual_target_pdf_hashes, + self.template_target_pdf_hashes, + "Hashes of downloaded pdfs (security target) do not match the template", + ) - self.assertTrue(dset['309ac2fd7f2dcf17'].state.report_txt_path.exists()) - self.assertTrue(dset['309ac2fd7f2dcf17'].state.st_txt_path.exists()) + self.assertTrue(dset["309ac2fd7f2dcf17"].state.report_txt_path.exists()) + self.assertTrue(dset["309ac2fd7f2dcf17"].state.st_txt_path.exists()) - self.assertAlmostEqual(dset['309ac2fd7f2dcf17'].state.st_txt_path.stat().st_size, - self.template_target_txt_path.stat().st_size, - delta=1000) + self.assertAlmostEqual( + dset["309ac2fd7f2dcf17"].state.st_txt_path.stat().st_size, + self.template_target_txt_path.stat().st_size, + delta=1000, + ) - self.assertAlmostEqual(dset['309ac2fd7f2dcf17'].state.report_txt_path.stat().st_size, - self.template_report_txt_path.stat().st_size, - delta=1000) + self.assertAlmostEqual( + dset["309ac2fd7f2dcf17"].state.report_txt_path.stat().st_size, + self.template_report_txt_path.stat().st_size, + delta=1000, + ) def test_cert_to_json(self): - with NamedTemporaryFile('w') as tmp: + with NamedTemporaryFile("w") as tmp: self.fictional_cert.to_json(tmp.name) - self.assertTrue(filecmp.cmp(self.test_data_dir / 'fictional_cert.json', - tmp.name), - 'The sample serialized to json differs from a template.') + self.assertTrue( + filecmp.cmp(self.test_data_dir / "fictional_cert.json", tmp.name), + "The sample serialized to json differs from a template.", + ) def test_dataset_to_json(self): - with NamedTemporaryFile('w') as tmp: + with NamedTemporaryFile("w") as tmp: self.template_dataset.to_json(tmp.name) - self.assertTrue(filecmp.cmp(self.test_data_dir / 'toy_dataset.json', - tmp.name), - 'The dataset serialized to json differs from a template.') + self.assertTrue( + filecmp.cmp(self.test_data_dir / "toy_dataset.json", tmp.name), + "The dataset serialized to json differs from a template.", + ) def test_cert_from_json(self): - self.assertEqual(self.fictional_cert, - CommonCriteriaCert.from_json(self.test_data_dir / 'fictional_cert.json'), - 'The sample serialized from json differs from a template.') + self.assertEqual( + self.fictional_cert, + CommonCriteriaCert.from_json(self.test_data_dir / "fictional_cert.json"), + "The sample serialized from json differs from a template.", + ) def test_dataset_from_json(self): - self.assertEqual(self.template_dataset, - CCDataset.from_json(self.test_data_dir / 'toy_dataset.json'), - 'The dataset serialized from json differs from a template.') + self.assertEqual( + self.template_dataset, + CCDataset.from_json(self.test_data_dir / "toy_dataset.json"), + "The dataset serialized from json differs from a template.", + ) def test_build_empty_dataset(self): with TemporaryDirectory() as tmp_dir: - dset = CCDataset({}, Path(tmp_dir), 'sample_dataset', 'sample dataset description') + dset = CCDataset({}, Path(tmp_dir), "sample_dataset", "sample dataset description") dset.get_certs_from_web(to_download=False, get_archived=False, get_active=False) - self.assertEqual(len(dset), 0, 'The dataset should contain 0 files.') + self.assertEqual(len(dset), 0, "The dataset should contain 0 files.") def test_build_dataset(self): with TemporaryDirectory() as tmp_dir: dataset_path = Path(tmp_dir) - os.mkdir(dataset_path / 'web') - shutil.copyfile(self.test_data_dir / 'cc_products_active.csv', dataset_path / 'web' / 'cc_products_active.csv') - shutil.copyfile(self.test_data_dir / 'cc_products_active.html', dataset_path / 'web' / 'cc_products_active.html') + os.mkdir(dataset_path / "web") + shutil.copyfile( + self.test_data_dir / "cc_products_active.csv", dataset_path / "web" / "cc_products_active.csv" + ) + shutil.copyfile( + self.test_data_dir / "cc_products_active.html", dataset_path / "web" / "cc_products_active.html" + ) - dset = CCDataset({}, dataset_path, 'sample_dataset', 'sample dataset description') - dset.get_certs_from_web(keep_metadata=False, - to_download=False, - get_archived=False, - get_active=True, - update_json=False) + dset = CCDataset({}, dataset_path, "sample_dataset", "sample dataset description") + dset.get_certs_from_web( + keep_metadata=False, to_download=False, get_archived=False, get_active=True, update_json=False + ) - self.assertEqual(len(os.listdir(dataset_path)), 0, - 'Meta files (csv, html) were not deleted properly albeit this was explicitly required.') + self.assertEqual( + len(os.listdir(dataset_path)), + 0, + "Meta files (csv, html) were not deleted properly albeit this was explicitly required.", + ) - self.assertEqual(len(dset), 2, 'The dataset should contain 2 files.') + self.assertEqual(len(dset), 2, "The dataset should contain 2 files.") - self.assertTrue(self.crt_one in dset, 'The dataset does not contain the template sample.') - self.assertEqual(dset, self.template_dataset, 'The loaded dataset does not match the template dataset.') + self.assertTrue(self.crt_one in dset, "The dataset does not contain the template sample.") + self.assertEqual(dset, self.template_dataset, "The loaded dataset does not match the template dataset.") def test_download_csv_html_files(self): with TemporaryDirectory() as tmp_dir: dataset_path = Path(tmp_dir) - dset = CCDataset({}, dataset_path, 'sample_dataset', 'sample dataset description') + dset = CCDataset({}, dataset_path, "sample_dataset", "sample dataset description") dset.download_csv_html_resources(get_active=True, get_archived=False) for x in dset.active_html_tuples: @@ -187,4 +235,6 @@ class TestCommonCriteriaOOP(TestCase): self.template_dataset.root_dir = tmp_dir self.template_dataset.process_protection_profiles() self.assertTrue(self.template_dataset.pp_dataset_path.exists()) - self.assertGreaterEqual(self.template_dataset.pp_dataset_path.stat().st_size, constants.MIN_CC_PP_DATASET_SIZE)
\ No newline at end of file + self.assertGreaterEqual( + self.template_dataset.pp_dataset_path.stat().st_size, constants.MIN_CC_PP_DATASET_SIZE + ) diff --git a/tests/test_cc_txt_processing.py b/tests/test_cc_txt_processing.py index 01b2212b..cc1cb23c 100644 --- a/tests/test_cc_txt_processing.py +++ b/tests/test_cc_txt_processing.py @@ -1,24 +1,26 @@ -import tests.data.test_cc_heuristics -from unittest import TestCase +import shutil +import tempfile from pathlib import Path from typing import ClassVar -import tempfile -import shutil +from unittest import TestCase +import tests.data.test_cc_heuristics from sec_certs.dataset.common_criteria import CCDataset from sec_certs.sample.common_criteria import CommonCriteriaCert -class TestCommonCriteriaHeuristics(TestCase): - dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / 'vulnerable_dataset.json' +class TestCommonCriteriaTextProcessing(TestCase): + dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / "vulnerable_dataset.json" # type: ignore # mypy issue #1422 data_dir_path: ClassVar[Path] = dataset_json_path.parent + tmp_dir: ClassVar[tempfile.TemporaryDirectory] + cc_dset: CCDataset @classmethod def setUpClass(cls) -> None: - cls.tmp_dir: ClassVar[tempfile.TemporaryDirectory] = tempfile.TemporaryDirectory() + cls.tmp_dir = tempfile.TemporaryDirectory() shutil.copytree(cls.data_dir_path, cls.tmp_dir.name, dirs_exist_ok=True) - cls.cc_dset: CCDataset = CCDataset.from_json(Path(cls.tmp_dir.name) / 'vulnerable_dataset.json') + cls.cc_dset = CCDataset.from_json(Path(cls.tmp_dir.name) / "vulnerable_dataset.json") cls.cc_dset.download_all_pdfs() cls.cc_dset.convert_all_pdfs() diff --git a/tests/test_fips_oop.py b/tests/test_fips_oop.py index 4d858edb..17b3325a 100644 --- a/tests/test_fips_oop.py +++ b/tests/test_fips_oop.py @@ -1,25 +1,24 @@ -from unittest import TestCase +import shutil from pathlib import Path from tempfile import TemporaryDirectory -from typing import Optional, Union, ClassVar, Final, List, Dict -import shutil +from typing import Dict, Final, List +from unittest import TestCase +import tests.data.test_fips_oop +from sec_certs.config.configuration import config from sec_certs.dataset.fips import FIPSDataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.config.configuration import config from tests.fips_test_utils import generate_html -import tests.data.test_fips_oop - def _set_up_dataset(td, certs): - dataset = FIPSDataset({}, Path(td), 'test_dataset', 'fips_test_dataset') - generate_html(certs, td + '/test_search.html') - dataset.get_certs_from_web(test=td + '/test_search.html', no_download_algorithms=True) + dataset = FIPSDataset({}, Path(td), "test_dataset", "fips_test_dataset") + generate_html(certs, td + "/test_search.html") + dataset.get_certs_from_web(test=td + "/test_search.html", no_download_algorithms=True) return dataset -def _set_up_dataset_for_full(td, certs, cpe_dset_path: Union[str, Path] = None, cve_dset_path: Union[str, Path] = None): +def _set_up_dataset_for_full(td, certs, cpe_dset_path: Path, cve_dset_path: Path): dataset = _set_up_dataset(td, certs) dataset.auxillary_datasets_dir.mkdir(exist_ok=True) @@ -31,25 +30,76 @@ def _set_up_dataset_for_full(td, certs, cpe_dset_path: Union[str, Path] = None, dataset.convert_all_pdfs() dataset.pdf_scan() dataset.extract_certs_from_tables(high_precision=True) - dataset.algorithms = FIPSAlgorithmDataset.from_json(Path(__file__).parent / 'data/test_fips_oop/algorithms.json') + dataset.algorithms = FIPSAlgorithmDataset.from_json(Path(__file__).parent / "data/test_fips_oop/algorithms.json") dataset.finalize_results(use_nist_cpe_matching_dict=False, perform_cpe_heuristics=False) return dataset class TestFipsOOP(TestCase): data_dir: Final[Path] = Path(tests.data.test_fips_oop.__path__[0]) - cpe_dset_path: Final[Path] = data_dir.parent / 'test_cc_heuristics/auxillary_datasets/cpe_dataset.json' - cve_dset_path: Final[Path] = data_dir.parent / 'test_cc_heuristics/auxillary_datasets/cve_dataset.json' + cpe_dset_path: Final[Path] = data_dir.parent / "test_cc_heuristics/auxillary_datasets/cpe_dataset.json" + cve_dset_path: Final[Path] = data_dir.parent / "test_cc_heuristics/auxillary_datasets/cve_dataset.json" certs_to_parse: Final[Dict[str, List[str]]] = { - 'microsoft': ['3095', '3651', '3093', '3090', '3197', '3196', '3089', '3195', '3480', '3615', '3194', '3091', '3690', '3644', '3527', '3094', '3544', '3096', '3092'], - 'redhat': ['2630', '2721', '2997', '2441', '2711', '2633', '2798', '3613', '3733', '2908', '2446', '2742', '2447'], - 'docusign': ['3850', '2779', '2860', '2665', '1883', '3518', '3141', '2590'], - 'referencing_openssl': ['3493', '3495', '3711', '3176', '3488', '3126', '3269', '3524', '3220', '2398', '3543', '2676', '3313', '3363', '3608', '3158'] + "microsoft": [ + "3095", + "3651", + "3093", + "3090", + "3197", + "3196", + "3089", + "3195", + "3480", + "3615", + "3194", + "3091", + "3690", + "3644", + "3527", + "3094", + "3544", + "3096", + "3092", + ], + "redhat": [ + "2630", + "2721", + "2997", + "2441", + "2711", + "2633", + "2798", + "3613", + "3733", + "2908", + "2446", + "2742", + "2447", + ], + "docusign": ["3850", "2779", "2860", "2665", "1883", "3518", "3141", "2590"], + "referencing_openssl": [ + "3493", + "3495", + "3711", + "3176", + "3488", + "3126", + "3269", + "3524", + "3220", + "2398", + "3543", + "2676", + "3313", + "3363", + "3608", + "3158", + ], } @classmethod def setUpClass(cls) -> None: - config.load(cls.data_dir.parent / 'settings_test.yaml') + config.load(cls.data_dir.parent / "settings_test.yaml") def test_size(self): for certs in self.certs_to_parse.values(): @@ -58,82 +108,88 @@ class TestFipsOOP(TestCase): self.assertEqual(len(dataset.certs), len(certs), "Wrong number of parsed certs") def test_connections_microsoft(self): - certs = self.certs_to_parse['microsoft'] + certs = self.certs_to_parse["microsoft"] with TemporaryDirectory() as tmp_dir: dataset = _set_up_dataset_for_full(tmp_dir, certs, self.cpe_dset_path, self.cve_dset_path) - self.assertEqual(set(dataset.certs['3095'].heuristics.connections), {'3093', '3096', '3094'}) - self.assertEqual(set(dataset.certs['3651'].heuristics.connections), {'3615'}) - self.assertEqual(set(dataset.certs['3093'].heuristics.connections), {'3090', '3091'}) - self.assertEqual(set(dataset.certs['3090'].heuristics.connections), {'3089'}) - self.assertEqual(set(dataset.certs['3197'].heuristics.connections), {x for x in ['3195', '3096', '3196', '3644', '3651']}) - self.assertEqual(set(dataset.certs['3196'].heuristics.connections), {x for x in ['3194', '3091', '3480', '3615']}) - self.assertEqual(set(dataset.certs['3089'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['3195'].heuristics.connections), {'3194', '3091', '3480'}) - self.assertEqual(set(dataset.certs['3480'].heuristics.connections), {'3089'}) - self.assertEqual(set(dataset.certs['3615'].heuristics.connections), {'3089'}) - self.assertEqual(set(dataset.certs['3194'].heuristics.connections), {'3089'}) - self.assertEqual(set(dataset.certs['3091'].heuristics.connections), {'3089'}) - self.assertEqual(set(dataset.certs['3690'].heuristics.connections), {'3644', '3196', '3651'}) - self.assertEqual(set(dataset.certs['3644'].heuristics.connections), {'3615'}) - self.assertEqual(set(dataset.certs['3527'].heuristics.connections), {'3090', '3091'}) - self.assertEqual(set(dataset.certs['3094'].heuristics.connections), {'3090', '3091'}) - self.assertEqual(set(dataset.certs['3544'].heuristics.connections), {'3093', '3096', '3527'}) - self.assertEqual(set(dataset.certs['3096'].heuristics.connections), {'3090', '3194', '3091', '3480'}) - self.assertEqual(set(dataset.certs['3092'].heuristics.connections), {'3093', '3195', '3096', '3644', '3651'}) + self.assertEqual(set(dataset.certs["3095"].heuristics.connections), {"3093", "3096", "3094"}) + self.assertEqual(set(dataset.certs["3651"].heuristics.connections), {"3615"}) + self.assertEqual(set(dataset.certs["3093"].heuristics.connections), {"3090", "3091"}) + self.assertEqual(set(dataset.certs["3090"].heuristics.connections), {"3089"}) + self.assertEqual( + set(dataset.certs["3197"].heuristics.connections), {x for x in ["3195", "3096", "3196", "3644", "3651"]} + ) + self.assertEqual( + set(dataset.certs["3196"].heuristics.connections), {x for x in ["3194", "3091", "3480", "3615"]} + ) + self.assertEqual(set(dataset.certs["3089"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["3195"].heuristics.connections), {"3194", "3091", "3480"}) + self.assertEqual(set(dataset.certs["3480"].heuristics.connections), {"3089"}) + self.assertEqual(set(dataset.certs["3615"].heuristics.connections), {"3089"}) + self.assertEqual(set(dataset.certs["3194"].heuristics.connections), {"3089"}) + self.assertEqual(set(dataset.certs["3091"].heuristics.connections), {"3089"}) + self.assertEqual(set(dataset.certs["3690"].heuristics.connections), {"3644", "3196", "3651"}) + self.assertEqual(set(dataset.certs["3644"].heuristics.connections), {"3615"}) + self.assertEqual(set(dataset.certs["3527"].heuristics.connections), {"3090", "3091"}) + self.assertEqual(set(dataset.certs["3094"].heuristics.connections), {"3090", "3091"}) + self.assertEqual(set(dataset.certs["3544"].heuristics.connections), {"3093", "3096", "3527"}) + self.assertEqual(set(dataset.certs["3096"].heuristics.connections), {"3090", "3194", "3091", "3480"}) + self.assertEqual( + set(dataset.certs["3092"].heuristics.connections), {"3093", "3195", "3096", "3644", "3651"} + ) def test_connections_redhat(self): - certs = self.certs_to_parse['redhat'] + certs = self.certs_to_parse["redhat"] with TemporaryDirectory() as tmp_dir: dataset = _set_up_dataset_for_full(tmp_dir, certs, self.cpe_dset_path, self.cve_dset_path) - self.assertEqual(set(dataset.certs['2630'].heuristics.connections), {'2441'}) - self.assertEqual(set(dataset.certs['2633'].heuristics.connections), {'2441'}) - self.assertEqual(set(dataset.certs['2441'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['2997'].heuristics.connections), {'2711'}) - self.assertEqual(set(dataset.certs['2446'].heuristics.connections), {'2441'}) - self.assertEqual(set(dataset.certs['2447'].heuristics.connections), {'2441'}) - self.assertEqual(set(dataset.certs['3733'].heuristics.connections), {'2441'}) - self.assertEqual(set(dataset.certs['2441'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['2711'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['2908'].heuristics.connections), {'2711'}) - self.assertEqual(set(dataset.certs['3613'].heuristics.connections), {'2997'}) - self.assertEqual(set(dataset.certs['2721'].heuristics.connections), {'2441', '2711'}) - self.assertEqual(set(dataset.certs['2798'].heuristics.connections), {'2721', '2711'}) - self.assertEqual(set(dataset.certs['2711'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['2997'].heuristics.connections), {'2711'}) - self.assertEqual(set(dataset.certs['2742'].heuristics.connections), {'2721', '2711'}) - self.assertEqual(set(dataset.certs['2721'].heuristics.connections), {'2441', '2711'}) + self.assertEqual(set(dataset.certs["2630"].heuristics.connections), {"2441"}) + self.assertEqual(set(dataset.certs["2633"].heuristics.connections), {"2441"}) + self.assertEqual(set(dataset.certs["2441"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["2997"].heuristics.connections), {"2711"}) + self.assertEqual(set(dataset.certs["2446"].heuristics.connections), {"2441"}) + self.assertEqual(set(dataset.certs["2447"].heuristics.connections), {"2441"}) + self.assertEqual(set(dataset.certs["3733"].heuristics.connections), {"2441"}) + self.assertEqual(set(dataset.certs["2441"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["2711"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["2908"].heuristics.connections), {"2711"}) + self.assertEqual(set(dataset.certs["3613"].heuristics.connections), {"2997"}) + self.assertEqual(set(dataset.certs["2721"].heuristics.connections), {"2441", "2711"}) + self.assertEqual(set(dataset.certs["2798"].heuristics.connections), {"2721", "2711"}) + self.assertEqual(set(dataset.certs["2711"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["2997"].heuristics.connections), {"2711"}) + self.assertEqual(set(dataset.certs["2742"].heuristics.connections), {"2721", "2711"}) + self.assertEqual(set(dataset.certs["2721"].heuristics.connections), {"2441", "2711"}) def test_docusign_chunk(self): - certs = self.certs_to_parse['docusign'] + certs = self.certs_to_parse["docusign"] with TemporaryDirectory() as tmp_dir: dataset = _set_up_dataset_for_full(tmp_dir, certs, self.cpe_dset_path, self.cve_dset_path) - self.assertEqual(set(dataset.certs['3850'].heuristics.connections), {'3518', '1883'}) - self.assertEqual(set(dataset.certs['2779'].heuristics.connections), {'1883'}) - self.assertEqual(set(dataset.certs['2860'].heuristics.connections), {'1883'}) - self.assertEqual(set(dataset.certs['2665'].heuristics.connections), {'1883'}) - self.assertEqual(set(dataset.certs['1883'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['3518'].heuristics.connections), {'1883'}) - self.assertEqual(set(dataset.certs['3141'].heuristics.connections), {'1883'}) - self.assertEqual(set(dataset.certs['2590'].heuristics.connections), {'1883'}) + self.assertEqual(set(dataset.certs["3850"].heuristics.connections), {"3518", "1883"}) + self.assertEqual(set(dataset.certs["2779"].heuristics.connections), {"1883"}) + self.assertEqual(set(dataset.certs["2860"].heuristics.connections), {"1883"}) + self.assertEqual(set(dataset.certs["2665"].heuristics.connections), {"1883"}) + self.assertEqual(set(dataset.certs["1883"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["3518"].heuristics.connections), {"1883"}) + self.assertEqual(set(dataset.certs["3141"].heuristics.connections), {"1883"}) + self.assertEqual(set(dataset.certs["2590"].heuristics.connections), {"1883"}) def test_openssl_chunk(self): - certs = self.certs_to_parse['referencing_openssl'] + certs = self.certs_to_parse["referencing_openssl"] with TemporaryDirectory() as tmp_dir: dataset = _set_up_dataset_for_full(tmp_dir, certs, self.cpe_dset_path, self.cve_dset_path) - self.assertEqual(set(dataset.certs['3493'].heuristics.connections), {'2398'}) - self.assertEqual(set(dataset.certs['3495'].heuristics.connections), {'2398'}) - self.assertEqual(set(dataset.certs['3711'].heuristics.connections), {'3220'}) - self.assertEqual(set(dataset.certs['3176'].heuristics.connections), {'2398'}) - self.assertEqual(set(dataset.certs['3488'].heuristics.connections), {'2398'}) - self.assertEqual(set(dataset.certs['3126'].heuristics.connections), {'3126', '2398'}) - self.assertEqual(set(dataset.certs['3269'].heuristics.connections), {'3269', '3220'}) - self.assertEqual(set(dataset.certs['3524'].heuristics.connections), {'3220'}) - self.assertEqual(set(dataset.certs['3220'].heuristics.connections), {'3220', '2398'}) - self.assertEqual(set(dataset.certs['2398'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['3543'].heuristics.connections), {'2398'}) - self.assertEqual(set(dataset.certs['2676'].heuristics.connections), {'2398'}) - self.assertEqual(set(dataset.certs['3313'].heuristics.connections), {'3313', '3220'}) - self.assertEqual(set(dataset.certs['3363'].heuristics.connections), set()) - self.assertEqual(set(dataset.certs['3608'].heuristics.connections), {'2398'}) - self.assertEqual(set(dataset.certs['3158'].heuristics.connections), {'2398'}) + self.assertEqual(set(dataset.certs["3493"].heuristics.connections), {"2398"}) + self.assertEqual(set(dataset.certs["3495"].heuristics.connections), {"2398"}) + self.assertEqual(set(dataset.certs["3711"].heuristics.connections), {"3220"}) + self.assertEqual(set(dataset.certs["3176"].heuristics.connections), {"2398"}) + self.assertEqual(set(dataset.certs["3488"].heuristics.connections), {"2398"}) + self.assertEqual(set(dataset.certs["3126"].heuristics.connections), {"3126", "2398"}) + self.assertEqual(set(dataset.certs["3269"].heuristics.connections), {"3269", "3220"}) + self.assertEqual(set(dataset.certs["3524"].heuristics.connections), {"3220"}) + self.assertEqual(set(dataset.certs["3220"].heuristics.connections), {"3220", "2398"}) + self.assertEqual(set(dataset.certs["2398"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["3543"].heuristics.connections), {"2398"}) + self.assertEqual(set(dataset.certs["2676"].heuristics.connections), {"2398"}) + self.assertEqual(set(dataset.certs["3313"].heuristics.connections), {"3313", "3220"}) + self.assertEqual(set(dataset.certs["3363"].heuristics.connections), set()) + self.assertEqual(set(dataset.certs["3608"].heuristics.connections), {"2398"}) + self.assertEqual(set(dataset.certs["3158"].heuristics.connections), {"2398"}) |