From 73b3b0c361f9545450fa188bec50606d64bb1afd Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Fri, 9 Dec 2022 17:10:19 +0100 Subject: flat -> src layout (#294) - Some mypy fixes - Flat layout -> src layout - Ditch `setup.py` and `setup.cfg` in favour of `pyproject.toml` - Non-pinned requirements moved from `requirements/*.in` to `pyproject.toml`--- .flake8 | 5 - .github/workflows/tests.yml | 2 + .gitignore | 13 +- CONTRIBUTING.md | 3 +- codecov.yml | 1 - docs/api/dataset.md | 2 +- docs/index.md | 2 + notebooks/cc/temporal_trends.ipynb | 5 +- notebooks/examples/fips_iut.ipynb | 19 +- notebooks/examples/fips_mip.ipynb | 4 +- pyproject.toml | 133 +- requirements/compile.sh | 6 +- requirements/dev_requirements.in | 20 - requirements/dev_requirements.txt | 431 +++-- requirements/requirements.in | 30 - requirements/requirements.txt | 319 ++-- requirements/test_requirements.in | 3 - requirements/test_requirements.txt | 327 +++- sec_certs/__init__.py | 7 - sec_certs/__main__.py | 6 - sec_certs/cert_rules.py | 248 --- sec_certs/cli.py | 211 --- sec_certs/config/__init__.py | 0 sec_certs/config/configuration.py | 44 - sec_certs/config/settings-schema.json | 182 --- sec_certs/config/settings.yaml | 59 - sec_certs/constants.py | 121 -- sec_certs/dataset/__init__.py | 22 - sec_certs/dataset/common_criteria.py | 1674 -------------------- sec_certs/dataset/cpe.py | 197 --- sec_certs/dataset/cve.py | 221 --- sec_certs/dataset/dataset.py | 572 ------- sec_certs/dataset/fips.py | 355 ----- sec_certs/dataset/fips_algorithm.py | 124 -- sec_certs/dataset/fips_iut.py | 66 - sec_certs/dataset/fips_mip.py | 66 - sec_certs/dataset/json_path_dataset.py | 46 - sec_certs/dataset/protection_profile.py | 98 -- sec_certs/model/__init__.py | 11 - sec_certs/model/cpe_matching.py | 404 ----- sec_certs/model/evaluation.py | 101 -- sec_certs/model/reference_finder.py | 223 --- sec_certs/model/sar_transformer.py | 159 -- sec_certs/model/transitive_vulnerability_finder.py | 149 -- sec_certs/rules.yaml | 1186 -------------- sec_certs/sample/__init__.py | 33 - sec_certs/sample/cc_certificate_id.py | 151 -- sec_certs/sample/cc_maintenance_update.py | 99 -- sec_certs/sample/certificate.py | 90 -- sec_certs/sample/common_criteria.py | 986 ------------ sec_certs/sample/cpe.py | 100 -- sec_certs/sample/cve.py | 185 --- sec_certs/sample/fips.py | 649 -------- sec_certs/sample/fips_algorithm.py | 50 - sec_certs/sample/fips_iut.py | 166 -- sec_certs/sample/fips_mip.py | 242 --- sec_certs/sample/protection_profile.py | 55 - sec_certs/sample/sar.py | 59 - sec_certs/serialization/__init__.py | 0 sec_certs/serialization/json.py | 145 -- sec_certs/serialization/pandas.py | 16 - sec_certs/utils/__init__.py | 0 sec_certs/utils/extract.py | 817 ---------- sec_certs/utils/helpers.py | 239 --- sec_certs/utils/pandas.py | 542 ------- sec_certs/utils/parallel_processing.py | 43 - sec_certs/utils/pdf.py | 275 ---- sec_certs/utils/sanitization.py | 51 - sec_certs/utils/tables.py | 62 - sec_certs/utils/tqdm.py | 9 - setup.py | 38 - src/sec_certs/__init__.py | 7 + src/sec_certs/__main__.py | 6 + src/sec_certs/cert_rules.py | 248 +++ src/sec_certs/cli.py | 211 +++ src/sec_certs/config/__init__.py | 0 src/sec_certs/config/configuration.py | 44 + src/sec_certs/config/settings-schema.json | 182 +++ src/sec_certs/config/settings.yaml | 59 + src/sec_certs/constants.py | 121 ++ src/sec_certs/dataset/__init__.py | 22 + src/sec_certs/dataset/common_criteria.py | 1674 ++++++++++++++++++++ src/sec_certs/dataset/cpe.py | 197 +++ src/sec_certs/dataset/cve.py | 221 +++ src/sec_certs/dataset/dataset.py | 572 +++++++ src/sec_certs/dataset/fips.py | 355 +++++ src/sec_certs/dataset/fips_algorithm.py | 124 ++ src/sec_certs/dataset/fips_iut.py | 66 + src/sec_certs/dataset/fips_mip.py | 66 + src/sec_certs/dataset/json_path_dataset.py | 46 + src/sec_certs/dataset/protection_profile.py | 98 ++ src/sec_certs/model/__init__.py | 11 + src/sec_certs/model/cpe_matching.py | 404 +++++ src/sec_certs/model/evaluation.py | 101 ++ src/sec_certs/model/reference_finder.py | 223 +++ src/sec_certs/model/sar_transformer.py | 159 ++ .../model/transitive_vulnerability_finder.py | 149 ++ src/sec_certs/rules.yaml | 1186 ++++++++++++++ src/sec_certs/sample/__init__.py | 33 + src/sec_certs/sample/cc_certificate_id.py | 151 ++ src/sec_certs/sample/cc_maintenance_update.py | 99 ++ src/sec_certs/sample/certificate.py | 90 ++ src/sec_certs/sample/common_criteria.py | 986 ++++++++++++ src/sec_certs/sample/cpe.py | 100 ++ src/sec_certs/sample/cve.py | 185 +++ src/sec_certs/sample/fips.py | 654 ++++++++ src/sec_certs/sample/fips_algorithm.py | 50 + src/sec_certs/sample/fips_iut.py | 166 ++ src/sec_certs/sample/fips_mip.py | 242 +++ src/sec_certs/sample/protection_profile.py | 55 + src/sec_certs/sample/sar.py | 59 + src/sec_certs/serialization/__init__.py | 0 src/sec_certs/serialization/json.py | 145 ++ src/sec_certs/serialization/pandas.py | 16 + src/sec_certs/utils/__init__.py | 0 src/sec_certs/utils/extract.py | 817 ++++++++++ src/sec_certs/utils/helpers.py | 239 +++ src/sec_certs/utils/pandas.py | 542 +++++++ src/sec_certs/utils/parallel_processing.py | 43 + src/sec_certs/utils/pdf.py | 275 ++++ src/sec_certs/utils/sanitization.py | 51 + src/sec_certs/utils/tables.py | 62 + src/sec_certs/utils/tqdm.py | 9 + 123 files changed, 12524 insertions(+), 12076 deletions(-) delete mode 100644 requirements/dev_requirements.in delete mode 100644 requirements/requirements.in delete mode 100644 requirements/test_requirements.in delete mode 100644 sec_certs/__init__.py delete mode 100644 sec_certs/__main__.py delete mode 100644 sec_certs/cert_rules.py delete mode 100644 sec_certs/cli.py delete mode 100644 sec_certs/config/__init__.py delete mode 100644 sec_certs/config/configuration.py delete mode 100644 sec_certs/config/settings-schema.json delete mode 100644 sec_certs/config/settings.yaml delete mode 100644 sec_certs/constants.py delete mode 100644 sec_certs/dataset/__init__.py delete mode 100644 sec_certs/dataset/common_criteria.py delete mode 100644 sec_certs/dataset/cpe.py delete mode 100644 sec_certs/dataset/cve.py delete mode 100644 sec_certs/dataset/dataset.py delete mode 100644 sec_certs/dataset/fips.py delete mode 100644 sec_certs/dataset/fips_algorithm.py delete mode 100644 sec_certs/dataset/fips_iut.py delete mode 100644 sec_certs/dataset/fips_mip.py delete mode 100644 sec_certs/dataset/json_path_dataset.py delete mode 100644 sec_certs/dataset/protection_profile.py delete mode 100644 sec_certs/model/__init__.py delete mode 100644 sec_certs/model/cpe_matching.py delete mode 100644 sec_certs/model/evaluation.py delete mode 100644 sec_certs/model/reference_finder.py delete mode 100644 sec_certs/model/sar_transformer.py delete mode 100644 sec_certs/model/transitive_vulnerability_finder.py delete mode 100644 sec_certs/rules.yaml delete mode 100644 sec_certs/sample/__init__.py delete mode 100644 sec_certs/sample/cc_certificate_id.py delete mode 100644 sec_certs/sample/cc_maintenance_update.py delete mode 100644 sec_certs/sample/certificate.py delete mode 100644 sec_certs/sample/common_criteria.py delete mode 100644 sec_certs/sample/cpe.py delete mode 100644 sec_certs/sample/cve.py delete mode 100644 sec_certs/sample/fips.py delete mode 100644 sec_certs/sample/fips_algorithm.py delete mode 100644 sec_certs/sample/fips_iut.py delete mode 100644 sec_certs/sample/fips_mip.py delete mode 100644 sec_certs/sample/protection_profile.py delete mode 100644 sec_certs/sample/sar.py delete mode 100644 sec_certs/serialization/__init__.py delete mode 100644 sec_certs/serialization/json.py delete mode 100644 sec_certs/serialization/pandas.py delete mode 100644 sec_certs/utils/__init__.py delete mode 100644 sec_certs/utils/extract.py delete mode 100644 sec_certs/utils/helpers.py delete mode 100644 sec_certs/utils/pandas.py delete mode 100644 sec_certs/utils/parallel_processing.py delete mode 100644 sec_certs/utils/pdf.py delete mode 100644 sec_certs/utils/sanitization.py delete mode 100644 sec_certs/utils/tables.py delete mode 100644 sec_certs/utils/tqdm.py delete mode 100644 setup.py create mode 100644 src/sec_certs/__init__.py create mode 100644 src/sec_certs/__main__.py create mode 100644 src/sec_certs/cert_rules.py create mode 100644 src/sec_certs/cli.py create mode 100644 src/sec_certs/config/__init__.py create mode 100644 src/sec_certs/config/configuration.py create mode 100644 src/sec_certs/config/settings-schema.json create mode 100644 src/sec_certs/config/settings.yaml create mode 100644 src/sec_certs/constants.py create mode 100644 src/sec_certs/dataset/__init__.py create mode 100644 src/sec_certs/dataset/common_criteria.py create mode 100644 src/sec_certs/dataset/cpe.py create mode 100644 src/sec_certs/dataset/cve.py create mode 100644 src/sec_certs/dataset/dataset.py create mode 100644 src/sec_certs/dataset/fips.py create mode 100644 src/sec_certs/dataset/fips_algorithm.py create mode 100644 src/sec_certs/dataset/fips_iut.py create mode 100644 src/sec_certs/dataset/fips_mip.py create mode 100644 src/sec_certs/dataset/json_path_dataset.py create mode 100644 src/sec_certs/dataset/protection_profile.py create mode 100644 src/sec_certs/model/__init__.py create mode 100644 src/sec_certs/model/cpe_matching.py create mode 100644 src/sec_certs/model/evaluation.py create mode 100644 src/sec_certs/model/reference_finder.py create mode 100644 src/sec_certs/model/sar_transformer.py create mode 100644 src/sec_certs/model/transitive_vulnerability_finder.py create mode 100644 src/sec_certs/rules.yaml create mode 100644 src/sec_certs/sample/__init__.py create mode 100644 src/sec_certs/sample/cc_certificate_id.py create mode 100644 src/sec_certs/sample/cc_maintenance_update.py create mode 100644 src/sec_certs/sample/certificate.py create mode 100644 src/sec_certs/sample/common_criteria.py create mode 100644 src/sec_certs/sample/cpe.py create mode 100644 src/sec_certs/sample/cve.py create mode 100644 src/sec_certs/sample/fips.py create mode 100644 src/sec_certs/sample/fips_algorithm.py create mode 100644 src/sec_certs/sample/fips_iut.py create mode 100644 src/sec_certs/sample/fips_mip.py create mode 100644 src/sec_certs/sample/protection_profile.py create mode 100644 src/sec_certs/sample/sar.py create mode 100644 src/sec_certs/serialization/__init__.py create mode 100644 src/sec_certs/serialization/json.py create mode 100644 src/sec_certs/serialization/pandas.py create mode 100644 src/sec_certs/utils/__init__.py create mode 100644 src/sec_certs/utils/extract.py create mode 100644 src/sec_certs/utils/helpers.py create mode 100644 src/sec_certs/utils/pandas.py create mode 100644 src/sec_certs/utils/parallel_processing.py create mode 100644 src/sec_certs/utils/pdf.py create mode 100644 src/sec_certs/utils/sanitization.py create mode 100644 src/sec_certs/utils/tables.py create mode 100644 src/sec_certs/utils/tqdm.py diff --git a/.flake8 b/.flake8 index 07ddab5e..2d24fa82 100644 --- a/.flake8 +++ b/.flake8 @@ -11,11 +11,6 @@ exclude = scratches, max-complexity = 10 -# Ignore complexity for CLI -per-file-ignores = - cc_cli.py: C901, - fips_cli.py: C901 - ignore = # line length, should be handleded by black E501, diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 255b6cac..8f2a037c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,6 +20,8 @@ jobs: run: | pip install -r requirements/requirements.txt pip install -r requirements/test_requirements.txt + - name: Install sec-certs + run: pip install -e . - name: Run tests run: pytest --cov=sec_certs tests - name: Code coverage upload diff --git a/.gitignore b/.gitignore index d9519043..d64264b0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ results*/ # version file by setuptools-scm -sec_certs/_version.py +src/sec_certs/_version.py # Byte-compiled / optimized / DLL files __pycache__/ @@ -116,20 +116,9 @@ virt/ # mypy .mypy_cache/ -# fips dataset -fips_dataset/ -fips_test_dataset/ - -# output pdfs -*connections* -*single* - # log cert_processing_log.txt ./cc_processing_log.txt -# Example script may dump classification report into the folder -examples/classification_report.json - # Experiment results produced by notebooks notebooks/cc/results/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d95b2ef9..958ba073 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,8 +15,7 @@ For complete list of system dependencies, see [docs/installation](https://seccer ### Requirements Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-tools). The main ideas are: -- List actual dependencies in `.in` files inside [requirements](https://github.com/crocs-muni/sec-certs/blob/main/requirements) folder without pinning them. -- Those dependencies are loaded into [setup.py](https://github.com/crocs-muni/sec-certs/blob/main/setup.py) file. +- List actual dependencies in [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) without pinning them. - Additionally, [compile.sh](https://github.com/crocs-muni/sec-certs/blob/main/requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. - Tests, linting and Docker all run against this reproducible environment of pinned requirements. diff --git a/codecov.yml b/codecov.yml index 5bf2bbe4..5c664ea1 100644 --- a/codecov.yml +++ b/codecov.yml @@ -5,4 +5,3 @@ coverage: ignore: - "test/**/*.py" - - "setup.py" \ No newline at end of file diff --git a/docs/api/dataset.md b/docs/api/dataset.md index 17ec943c..27a77e79 100644 --- a/docs/api/dataset.md +++ b/docs/api/dataset.md @@ -14,7 +14,7 @@ The examples related to this package can be found at [common criteria notebook]( ## CCDataset ```{eval-rst} -.. currentmodule:: sec_certs.dataset +.. currentmodule:: sec_certs.dataset.dataset .. autoclass:: Dataset :members: ``` diff --git a/docs/index.md b/docs/index.md index c52706c2..f9d58fb6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,6 +46,8 @@ configuration.md notebooks/examples/common_criteria.ipynb notebooks/examples/fips.ipynb notebooks/examples/model.ipynb +notebooks/examples/fips_iut.ipynb +notebooks/examples/fips_mip.ipynb ``` ```{toctree} diff --git a/notebooks/cc/temporal_trends.ipynb b/notebooks/cc/temporal_trends.ipynb index d7d69ca9..754d494e 100644 --- a/notebooks/cc/temporal_trends.ipynb +++ b/notebooks/cc/temporal_trends.ipynb @@ -91,11 +91,10 @@ "data = [n_certs.loc[n_certs.popular_categories == c, \"size\"].tolist()[:-1] for c in cats]\n", "\n", "palette = sns.color_palette(\"Spectral\", 5).as_hex()\n", - "colors = ','.join(palette)\n", "\n", - "plt.style.use(\"seaborn-white\")\n", + "plt.style.use(\"seaborn-v0_8-white\")\n", "plt.figure(figsize=(7,4))\n", - "plt.stackplot(years, data, labels=cats, colors=colors)\n", + "plt.stackplot(years, data, labels=cats, colors=palette)\n", "plt.legend(loc='upper left')\n", "plt.ylabel(\"Number of issued certificates\")\n", "plt.xlabel(\"Year of certification\")\n", diff --git a/notebooks/examples/fips_iut.ipynb b/notebooks/examples/fips_iut.ipynb index 77138282..506c114f 100644 --- a/notebooks/examples/fips_iut.ipynb +++ b/notebooks/examples/fips_iut.ipynb @@ -4,9 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Parse FIPS 'Implementation Under Test' pages.\n", + "## FIPS 'Implementation Under Test'\n", "\n", - "To use, download pages from the URL: https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/IUT-List\n", + "Functionality to parse FIPS 'Implementation Under Test' webpage. To use, download pages from the URL: [https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/IUT-List](https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/IUT-List)\n", "into a `directory` and name them `fips_iut_.html`.\n", "\n", "Then run the cells below" @@ -35,10 +35,21 @@ } ], "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('venv': venv)", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "name": "python", + "version": "3.8.13" }, - "orig_nbformat": 4 + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "a5b8c5b127d2cfe5bc3a1c933e197485eb9eba25154c3661362401503b4ef9d4" + } + } }, "nbformat": 4, "nbformat_minor": 2 diff --git a/notebooks/examples/fips_mip.ipynb b/notebooks/examples/fips_mip.ipynb index 7955a32f..f46ba23a 100644 --- a/notebooks/examples/fips_mip.ipynb +++ b/notebooks/examples/fips_mip.ipynb @@ -4,9 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Parse FIPS 'Modules In Process' pages.\n", + "## FIPS 'Modules In Process'\n", "\n", - "To use, download pages from the URL: https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/Modules-In-Process-List into a `directory` and name them `fips_mip_.html`.\n", + "Functionality to parse FIPS 'Modules In Process' webpage. To use, download pages from the URL: [https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/Modules-In-Process-List](https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/Modules-In-Process-List) into a `directory` and name them `fips_mip_.html`.\n", "\n", "Then run the following cells" ] diff --git a/pyproject.toml b/pyproject.toml index 2804889f..266b4dd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,111 @@ [build-system] -requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2", "wheel"] + requires = ["setuptools>=61.0", "setuptools_scm[toml]>=6.2", "wheel"] + build-backend = "setuptools.build_meta" + +[project] + name = "sec-certs" + authors = [ + { name = "Adam Janovsky", email = "adamjanovsky@mail.muni.cz" }, + { name = "Jan Jancar" }, + { name = "Petr Svenda" }, + { name = "Jiri Michalik" }, + { name = "Stanislav Bobon" }, + ] + description = "A tool for data scraping and analysis of security certificates from Common Criteria and FIPS 140-2/3 frameworks" + readme = "README.md" + license = { "text" = "MIT" } + classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + "Topic :: Security", + "Topic :: Security :: Cryptography", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + ] + requires-python = ">=3.8" + dynamic = ["version"] + dependencies = [ + "beautifulsoup4", + "billiard", + "click", + "html5lib", + "jsonschema", + "lxml", + "matplotlib", + "numpy", + "pandas", + "pdftotext", + "pikepdf", + "Pillow>=9.2.0", + "PyPDF2", + "python-dateutil", + "PyYAML", + "rapidfuzz", + "requests", + "scikit-learn", + "tabula-py", + "tqdm", + "setuptools-scm", + "ipykernel", + "ipywidgets", + "spacy", + "en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1.tar.gz", + "pkgconfig", + "seaborn", + "pySankeyBeta", + "scipy>=1.9.0", + "networkx", + ] + + [project.optional-dependencies] + dev = [ + "mypy", + "types-PyYAML", + "types-python-dateutil", + "types-requests", + "pytest", + "pytest-cov", + "pytest-monitor", + "pytest-profiling", + "black", + "isort", + "flake8", + "pre-commit", + "pip-tools", + "sphinx", + "myst-nb>=0.14", + "sphinx-book-theme", + "sphinx-design", + "sphinx-copybutton", + "pyupgrade", + "flake8-future-annotations", + "ipython!=8.7.0", + ] + test = ["pytest", "coverage", "pytest-cov"] + + [project.urls] + Homepage = "https://seccerts.org" + GitHub = "https://github.com/crocs-muni/sec-certs/" + Documentation = "https://seccerts.org/docs" + + [project.scripts] + sec-certs = "sec_certs.cli:main" + +[tool.setuptools.package-data] + "*" = ["*.yaml", "*.json"] + [tool.setuptools_scm] -write_to = "sec_certs/_version.py" -version_scheme = "no-guess-dev" + write_to = "src/sec_certs/_version.py" + version_scheme = "no-guess-dev" [tool.black] -line-length = 120 -exclude = ''' + line-length = 120 + exclude = ''' /( \.git | \.mypy_cache @@ -19,23 +117,24 @@ exclude = ''' | buck-out | build | dist + | src/sec_certs/_version.py )/ ''' [tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -line_length = 120 -skip = ["certsvenv", "build"] + multi_line_output = 3 + include_trailing_comma = true + force_grid_wrap = 0 + use_parentheses = true + ensure_newline_before_comments = true + line_length = 120 + skip = ["certsvenv", "build"] [tool.mypy] -plugins = ["numpy.typing.mypy_plugin"] -ignore_missing_imports = true -exclude = "build/" + plugins = ["numpy.typing.mypy_plugin"] + ignore_missing_imports = true + exclude = "build/" [tool.pytest.ini_options] -markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"] -addopts = "--cov sec_certs" + markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"] + addopts = "--cov sec_certs" diff --git a/requirements/compile.sh b/requirements/compile.sh index 8910b72d..70783aee 100755 --- a/requirements/compile.sh +++ b/requirements/compile.sh @@ -2,6 +2,6 @@ # See CONTRIBUTING.md for description -pip-compile requirements.in --no-header -pip-compile dev_requirements.in --no-header -pip-compile test_requirements.in --no-header +pip-compile --no-header --resolver=backtracking --output-file=requirements.txt ./../pyproject.toml +pip-compile --no-header --resolver=backtracking --extra dev -o dev_requirements.txt ./../pyproject.toml +pip-compile --no-header --resolver=backtracking --extra test -o test_requirements.txt ./../pyproject.toml diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in deleted file mode 100644 index 5071c08e..00000000 --- a/requirements/dev_requirements.in +++ /dev/null @@ -1,20 +0,0 @@ -mypy -types-PyYAML -types-python-dateutil -types-requests -pytest -pytest-cov -pytest-monitor -pytest-profiling -black -isort -flake8 -pre-commit -pip-tools -sphinx -myst-nb>=0.14 -sphinx-book-theme -sphinx-design -sphinx-copybutton -pyupgrade -flake8-future-annotations diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 2002ded7..e174d11b 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -1,172 +1,300 @@ alabaster==0.7.12 # via sphinx -asttokens==2.0.5 +appnope==0.1.3 + # via + # ipykernel + # ipython +asttokens==2.2.1 # via stack-data -attrs==21.4.0 +attrs==22.1.0 # via # jsonschema # jupyter-cache - # markdown-it-py # pytest -babel==2.10.1 +babel==2.11.0 # via sphinx backcall==0.2.0 # via ipython beautifulsoup4==4.11.1 - # via pydata-sphinx-theme -black==22.3.0 - # via -r dev_requirements.in + # via + # pydata-sphinx-theme + # sec-certs (./../pyproject.toml) +billiard==4.0.2 + # via sec-certs (./../pyproject.toml) +black==22.10.0 + # via sec-certs (./../pyproject.toml) +blis==0.7.9 + # via thinc +build==0.9.0 + # via pip-tools +catalogue==2.0.8 + # via + # spacy + # srsly + # thinc certifi==2022.12.7 # via requests cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests -click==8.1.2 +click==8.1.3 # via # black # jupyter-cache # pip-tools -coverage[toml]==6.3.2 + # sec-certs (./../pyproject.toml) + # typer +comm==0.1.2 + # via ipykernel +confection==0.0.3 + # via thinc +contourpy==1.0.6 + # via matplotlib +coverage[toml]==6.5.0 # via pytest-cov -debugpy==1.6.0 +cycler==0.11.0 + # via matplotlib +cymem==2.0.7 + # via + # preshed + # spacy + # thinc +debugpy==1.6.4 # via ipykernel decorator==5.1.1 # via ipython -distlib==0.3.4 +deprecation==2.1.0 + # via pikepdf +distlib==0.3.6 # via virtualenv +distro==1.8.0 + # via tabula-py docutils==0.17.1 # via - # myst-nb # myst-parser # pydata-sphinx-theme # sphinx - # sphinx-togglebutton +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1.tar.gz + # via sec-certs (./../pyproject.toml) entrypoints==0.4 # via jupyter-client -executing==0.8.3 +exceptiongroup==1.0.4 + # via pytest +executing==1.2.0 # via stack-data -fastjsonschema==2.15.3 +fastjsonschema==2.16.2 # via nbformat -filelock==3.6.0 +filelock==3.8.2 # via virtualenv -flake8==4.0.1 +flake8==6.0.0 # via - # -r dev_requirements.in # flake8-future-annotations + # sec-certs (./../pyproject.toml) flake8-future-annotations==1.0.0 - # via -r dev_requirements.in -gprof2dot==2021.2.21 + # via sec-certs (./../pyproject.toml) +fonttools==4.38.0 + # via matplotlib +gprof2dot==2022.7.29 # via pytest-profiling -greenlet==1.1.2 +greenlet==2.0.1 # via sqlalchemy -identify==2.4.12 +html5lib==1.1 + # via sec-certs (./../pyproject.toml) +identify==2.5.9 # via pre-commit -idna==3.3 +idna==3.4 # via requests -imagesize==1.3.0 +imagesize==1.4.1 # via sphinx -importlib-metadata==4.11.3 +importlib-metadata==5.1.0 # via # jupyter-cache # myst-nb + # sphinx +importlib-resources==5.10.1 + # via jsonschema iniconfig==1.1.1 # via pytest -ipykernel==6.13.0 - # via myst-nb -ipython==8.2.0 +ipykernel==6.19.1 + # via + # ipywidgets + # myst-nb + # sec-certs (./../pyproject.toml) +ipython==8.6.0 # via # ipykernel + # ipywidgets # myst-nb + # sec-certs (./../pyproject.toml) +ipywidgets==8.0.3 + # via sec-certs (./../pyproject.toml) isort==5.10.1 - # via -r dev_requirements.in -jedi==0.18.1 + # via sec-certs (./../pyproject.toml) +jedi==0.18.2 # via ipython -jinja2==3.1.1 +jinja2==3.1.2 # via # myst-parser + # spacy # sphinx -jsonschema==4.4.0 - # via nbformat +joblib==1.2.0 + # via scikit-learn +jsonschema==4.17.3 + # via + # nbformat + # sec-certs (./../pyproject.toml) jupyter-cache==0.5.0 # via myst-nb -jupyter-client==7.2.2 +jupyter-client==7.4.8 # via # ipykernel # nbclient -jupyter-core==4.10.0 +jupyter-core==5.1.0 # via # jupyter-client # nbformat -markdown-it-py==1.1.0 +jupyterlab-widgets==3.0.4 + # via ipywidgets +kiwisolver==1.4.4 + # via matplotlib +langcodes==3.3.0 + # via spacy +lxml==4.9.1 + # via + # pikepdf + # sec-certs (./../pyproject.toml) +markdown-it-py==2.1.0 # via # mdit-py-plugins # myst-parser markupsafe==2.1.1 # via jinja2 -matplotlib-inline==0.1.3 +matplotlib==3.6.2 + # via + # pysankeybeta + # seaborn + # sec-certs (./../pyproject.toml) +matplotlib-inline==0.1.6 # via # ipykernel # ipython -mccabe==0.6.1 +mccabe==0.7.0 # via flake8 -mdit-py-plugins==0.3.0 +mdit-py-plugins==0.3.3 # via myst-parser -memory-profiler==0.60.0 +mdurl==0.1.2 + # via markdown-it-py +memory-profiler==0.61.0 # via pytest-monitor -mypy==0.942 - # via -r dev_requirements.in +murmurhash==1.0.9 + # via + # preshed + # spacy + # thinc +mypy==0.991 + # via sec-certs (./../pyproject.toml) mypy-extensions==0.4.3 # via # black # mypy -myst-nb==0.14.0 - # via -r dev_requirements.in -myst-parser==0.17.2 +myst-nb==0.17.1 + # via sec-certs (./../pyproject.toml) +myst-parser==0.18.1 # via myst-nb nbclient==0.5.13 - # via jupyter-cache -nbformat==5.3.0 + # via + # jupyter-cache + # myst-nb +nbformat==5.7.0 # via # jupyter-cache # myst-nb # nbclient -nest-asyncio==1.5.5 +nest-asyncio==1.5.6 # via # ipykernel # jupyter-client # nbclient -nodeenv==1.6.0 +networkx==2.8.8 + # via sec-certs (./../pyproject.toml) +nodeenv==1.7.0 # via pre-commit -packaging==21.3 +numpy==1.23.5 + # via + # blis + # contourpy + # matplotlib + # pandas + # pysankeybeta + # scikit-learn + # scipy + # seaborn + # sec-certs (./../pyproject.toml) + # spacy + # tabula-py + # thinc +packaging==22.0 # via + # build + # deprecation # ipykernel + # matplotlib + # pikepdf # pydata-sphinx-theme # pytest + # setuptools-scm + # spacy # sphinx +pandas==1.5.2 + # via + # pysankeybeta + # seaborn + # sec-certs (./../pyproject.toml) + # tabula-py parso==0.8.3 # via jedi -pathspec==0.9.0 +pathspec==0.10.2 # via black -pep517==0.12.0 - # via pip-tools +pathy==0.10.1 + # via spacy +pdftotext==2.2.2 + # via sec-certs (./../pyproject.toml) +pep517==0.13.0 + # via build pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.5.1 - # via -r dev_requirements.in -platformdirs==2.5.1 +pikepdf==6.2.5 + # via sec-certs (./../pyproject.toml) +pillow==9.3.0 + # via + # matplotlib + # pikepdf + # sec-certs (./../pyproject.toml) +pip-tools==6.11.0 + # via sec-certs (./../pyproject.toml) +pkgconfig==1.5.5 + # via sec-certs (./../pyproject.toml) +pkgutil-resolve-name==1.3.10 + # via jsonschema +platformdirs==2.6.0 # via # black + # jupyter-core # virtualenv pluggy==1.0.0 # via pytest -pre-commit==2.18.1 - # via -r dev_requirements.in -prompt-toolkit==3.0.29 +pre-commit==2.20.0 + # via sec-certs (./../pyproject.toml) +preshed==3.0.8 + # via + # spacy + # thinc +prompt-toolkit==3.0.36 # via ipython -psutil==5.9.0 +psutil==5.9.4 # via # ipykernel # memory-profiler @@ -175,81 +303,120 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -py==1.11.0 - # via pytest -pycodestyle==2.8.0 +pycodestyle==2.10.0 # via flake8 +pydantic==1.10.2 + # via + # confection + # spacy + # thinc pydata-sphinx-theme==0.8.1 # via sphinx-book-theme -pyflakes==2.4.0 +pyflakes==3.0.1 # via flake8 -pygments==2.12.0 +pygments==2.13.0 # via # ipython # sphinx -pyparsing==3.0.7 - # via packaging -pyrsistent==0.18.1 +pyparsing==3.0.9 + # via matplotlib +pypdf2==2.11.2 + # via sec-certs (./../pyproject.toml) +pyrsistent==0.19.2 # via jsonschema -pytest==7.1.1 +pysankeybeta==1.4.0 + # via sec-certs (./../pyproject.toml) +pytest==7.2.0 # via - # -r dev_requirements.in # pytest-cov # pytest-monitor # pytest-profiling -pytest-cov==3.0.0 - # via -r dev_requirements.in -pytest-monitor==1.6.3 - # via -r dev_requirements.in + # sec-certs (./../pyproject.toml) +pytest-cov==4.0.0 + # via sec-certs (./../pyproject.toml) +pytest-monitor==1.6.5 + # via sec-certs (./../pyproject.toml) pytest-profiling==1.7.0 - # via -r dev_requirements.in + # via sec-certs (./../pyproject.toml) python-dateutil==2.8.2 - # via jupyter-client -pytz==2022.1 - # via babel -pyupgrade==3.2.3 - # via -r dev_requirements.in + # via + # jupyter-client + # matplotlib + # pandas + # sec-certs (./../pyproject.toml) +pytz==2022.6 + # via + # babel + # pandas +pyupgrade==3.3.1 + # via sec-certs (./../pyproject.toml) pyyaml==6.0 # via # jupyter-cache # myst-nb # myst-parser # pre-commit + # sec-certs (./../pyproject.toml) # sphinx-book-theme -pyzmq==22.3.0 - # via jupyter-client -requests==2.27.1 +pyzmq==24.0.1 + # via + # ipykernel + # jupyter-client +rapidfuzz==2.13.3 + # via sec-certs (./../pyproject.toml) +requests==2.28.1 # via # pytest-monitor + # sec-certs (./../pyproject.toml) + # spacy # sphinx +scikit-learn==1.2.0 + # via sec-certs (./../pyproject.toml) +scipy==1.9.3 + # via + # scikit-learn + # sec-certs (./../pyproject.toml) +seaborn==0.12.1 + # via + # pysankeybeta + # sec-certs (./../pyproject.toml) +setuptools-scm==7.0.5 + # via sec-certs (./../pyproject.toml) six==1.16.0 # via # asttokens + # html5lib # pytest-profiling # python-dateutil - # virtualenv +smart-open==6.2.0 + # via pathy snowballstemmer==2.2.0 # via sphinx soupsieve==2.3.2.post1 # via beautifulsoup4 +spacy==3.4.3 + # via + # en-core-web-sm + # sec-certs (./../pyproject.toml) +spacy-legacy==3.0.10 + # via spacy +spacy-loggers==1.0.4 + # via spacy sphinx==4.5.0 # via - # -r dev_requirements.in # myst-nb # myst-parser # pydata-sphinx-theme + # sec-certs (./../pyproject.toml) # sphinx-book-theme # sphinx-copybutton # sphinx-design - # sphinx-togglebutton -sphinx-book-theme==0.3.2 - # via -r dev_requirements.in -sphinx-copybutton==0.5.0 - # via -r dev_requirements.in -sphinx-design==0.1.0 - # via -r dev_requirements.in -sphinx-togglebutton==0.3.1 - # via myst-nb +sphinx-book-theme==0.3.3 + # via sec-certs (./../pyproject.toml) +sphinx-copybutton==0.5.1 + # via sec-certs (./../pyproject.toml) +sphinx-design==0.3.0 + # via sec-certs (./../pyproject.toml) sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -262,61 +429,97 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy==1.4.35 +sqlalchemy==1.4.44 # via jupyter-cache -stack-data==0.2.0 +srsly==2.4.5 + # via + # confection + # spacy + # thinc +stack-data==0.6.2 # via ipython -tabulate==0.8.9 +tabula-py==2.6.0 + # via sec-certs (./../pyproject.toml) +tabulate==0.9.0 # via jupyter-cache +thinc==8.1.5 + # via spacy +threadpoolctl==3.1.0 + # via scikit-learn tokenize-rt==5.0.0 # via pyupgrade toml==0.10.2 # via pre-commit tomli==2.0.1 # via + # black + # build # coverage # mypy # pep517 # pytest -tornado==6.1 + # setuptools-scm +tornado==6.2 # via # ipykernel # jupyter-client -traitlets==5.1.1 +tqdm==4.64.1 + # via + # sec-certs (./../pyproject.toml) + # spacy +traitlets==5.6.0 # via + # comm # ipykernel # ipython + # ipywidgets # jupyter-client # jupyter-core # matplotlib-inline # nbclient # nbformat -types-python-dateutil==2.8.10 - # via -r dev_requirements.in -types-pyyaml==6.0.5 - # via -r dev_requirements.in -types-requests==2.27.16 - # via -r dev_requirements.in -types-urllib3==1.26.11 +typer==0.7.0 + # via + # pathy + # spacy +types-python-dateutil==2.8.19.4 + # via sec-certs (./../pyproject.toml) +types-pyyaml==6.0.12.2 + # via sec-certs (./../pyproject.toml) +types-requests==2.28.11.5 + # via sec-certs (./../pyproject.toml) +types-urllib3==1.26.25.4 # via types-requests -typing-extensions==4.1.1 +typing-extensions==4.4.0 # via # mypy # myst-nb # myst-parser -urllib3==1.26.9 + # pydantic + # pypdf2 + # setuptools-scm +urllib3==1.26.13 # via requests -virtualenv==20.14.0 +virtualenv==20.17.1 # via pre-commit +wasabi==0.10.1 + # via + # spacy + # thinc wcwidth==0.2.5 # via prompt-toolkit -wheel==0.37.1 +webencodings==0.5.1 + # via html5lib +wheel==0.38.4 # via # pip-tools # pytest-monitor - # sphinx-togglebutton -zipp==3.8.0 - # via importlib-metadata +widgetsnbextension==4.0.4 + # via ipywidgets +zipp==3.11.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/requirements.in b/requirements/requirements.in deleted file mode 100644 index 8b66cc67..00000000 --- a/requirements/requirements.in +++ /dev/null @@ -1,30 +0,0 @@ -beautifulsoup4 -billiard -click -html5lib -jsonschema -lxml -matplotlib -numpy -pandas -pdftotext -pikepdf -Pillow>=9.2.0 -PyPDF2 -python-dateutil -PyYAML -rapidfuzz -requests -scikit-learn -tabula-py -tqdm -setuptools-scm -ipykernel -ipywidgets -spacy -en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1.tar.gz -pkgconfig -seaborn -pySankeyBeta -scipy>=1.9.0 -networkx \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 1947ab3a..91411c35 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,366 +1,303 @@ -argon2-cffi==21.3.0 - # via notebook -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -asttokens==2.0.5 +appnope==0.1.3 + # via + # ipykernel + # ipython +asttokens==2.2.1 # via stack-data -attrs==21.4.0 +attrs==22.1.0 # via jsonschema backcall==0.2.0 # via ipython -beautifulsoup4==4.10.0 - # via - # -r requirements.in - # nbconvert -billiard==3.6.4.0 - # via -r requirements.in -bleach==5.0.0 - # via nbconvert +beautifulsoup4==4.11.1 + # via sec-certs (./../pyproject.toml) +billiard==4.0.2 + # via sec-certs (./../pyproject.toml) blis==0.7.9 # via thinc -catalogue==2.0.7 +catalogue==2.0.8 # via # spacy # srsly # thinc certifi==2022.12.7 # via requests -cffi==1.15.0 - # via argon2-cffi-bindings -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests -click==8.1.2 +click==8.1.3 # via - # -r requirements.in + # sec-certs (./../pyproject.toml) # typer +comm==0.1.2 + # via ipykernel confection==0.0.3 # via thinc +contourpy==1.0.6 + # via matplotlib cycler==0.11.0 # via matplotlib -cymem==2.0.6 +cymem==2.0.7 # via # preshed # spacy # thinc -debugpy==1.6.0 +debugpy==1.6.4 # via ipykernel decorator==5.1.1 # via ipython -defusedxml==0.7.1 - # via nbconvert -distro==1.7.0 +deprecation==2.1.0 + # via pikepdf +distro==1.8.0 # via tabula-py -en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1.tar.gz - # via -r requirements.in +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1.tar.gz + # via sec-certs (./../pyproject.toml) entrypoints==0.4 - # via - # jupyter-client - # nbconvert -executing==0.8.3 + # via jupyter-client +executing==1.2.0 # via stack-data -fastjsonschema==2.15.3 - # via nbformat -fonttools==4.31.2 +fonttools==4.38.0 # via matplotlib html5lib==1.1 - # via -r requirements.in -idna==3.3 + # via sec-certs (./../pyproject.toml) +idna==3.4 # via requests -ipykernel==6.12.1 +importlib-resources==5.10.1 + # via jsonschema +ipykernel==6.19.1 # via - # -r requirements.in # ipywidgets - # notebook -ipython==8.2.0 + # sec-certs (./../pyproject.toml) +ipython==8.7.0 # via # ipykernel # ipywidgets -ipython-genutils==0.2.0 - # via - # ipywidgets - # notebook -ipywidgets==7.7.0 - # via -r requirements.in -jarowinkler==1.0.2 - # via rapidfuzz -jedi==0.18.1 +ipywidgets==8.0.3 + # via sec-certs (./../pyproject.toml) +jedi==0.18.2 # via ipython -jinja2==3.1.1 - # via - # nbconvert - # notebook - # spacy +jinja2==3.1.2 + # via spacy joblib==1.2.0 # via scikit-learn -jsonschema==4.4.0 - # via - # -r requirements.in - # nbformat -jupyter-client==7.2.1 - # via - # ipykernel - # nbclient - # notebook -jupyter-core==4.9.2 - # via - # jupyter-client - # nbconvert - # nbformat - # notebook -jupyterlab-pygments==0.2.2 - # via nbconvert -jupyterlab-widgets==1.1.0 +jsonschema==4.17.3 + # via sec-certs (./../pyproject.toml) +jupyter-client==7.4.8 + # via ipykernel +jupyter-core==5.1.0 + # via jupyter-client +jupyterlab-widgets==3.0.4 # via ipywidgets -kiwisolver==1.4.2 +kiwisolver==1.4.4 # via matplotlib langcodes==3.3.0 # via spacy lxml==4.9.1 # via - # -r requirements.in - # nbconvert # pikepdf + # sec-certs (./../pyproject.toml) markupsafe==2.1.1 + # via jinja2 +matplotlib==3.6.2 # via - # jinja2 - # nbconvert -matplotlib==3.5.1 - # via - # -r requirements.in # pysankeybeta # seaborn -matplotlib-inline==0.1.3 + # sec-certs (./../pyproject.toml) +matplotlib-inline==0.1.6 # via # ipykernel # ipython -mistune==0.8.4 - # via nbconvert -murmurhash==1.0.7 +murmurhash==1.0.9 # via # preshed # spacy # thinc -nbclient==0.6.0 - # via nbconvert -nbconvert==6.5.1 - # via notebook -nbformat==5.3.0 - # via - # ipywidgets - # nbclient - # nbconvert - # notebook -nest-asyncio==1.5.5 +nest-asyncio==1.5.6 # via # ipykernel # jupyter-client - # nbclient - # notebook -networkx==2.8.7 - # via -r requirements.in -notebook==6.4.12 - # via widgetsnbextension -numpy==1.22.3 +networkx==2.8.8 + # via sec-certs (./../pyproject.toml) +numpy==1.23.5 # via - # -r requirements.in # blis + # contourpy # matplotlib # pandas # pysankeybeta # scikit-learn # scipy # seaborn + # sec-certs (./../pyproject.toml) # spacy # tabula-py # thinc -packaging==21.3 +packaging==22.0 # via + # deprecation # ipykernel # matplotlib - # nbconvert # pikepdf # setuptools-scm # spacy -pandas==1.4.2 +pandas==1.5.2 # via - # -r requirements.in # pysankeybeta # seaborn + # sec-certs (./../pyproject.toml) # tabula-py -pandocfilters==1.5.0 - # via nbconvert parso==0.8.3 # via jedi -pathy==0.6.1 +pathy==0.10.1 # via spacy pdftotext==2.2.2 - # via -r requirements.in + # via sec-certs (./../pyproject.toml) pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pikepdf==5.1.1 - # via -r requirements.in +pikepdf==6.2.5 + # via sec-certs (./../pyproject.toml) pillow==9.3.0 # via - # -r requirements.in # matplotlib # pikepdf + # sec-certs (./../pyproject.toml) pkgconfig==1.5.5 - # via -r requirements.in -preshed==3.0.6 + # via sec-certs (./../pyproject.toml) +pkgutil-resolve-name==1.3.10 + # via jsonschema +platformdirs==2.6.0 + # via jupyter-core +preshed==3.0.8 # via # spacy # thinc -prometheus-client==0.14.1 - # via notebook -prompt-toolkit==3.0.29 +prompt-toolkit==3.0.36 # via ipython -psutil==5.9.0 +psutil==5.9.4 # via ipykernel ptyprocess==0.7.0 - # via - # pexpect - # terminado + # via pexpect pure-eval==0.2.2 # via stack-data -pycparser==2.21 - # via cffi -pydantic==1.8.2 +pydantic==1.10.2 # via # confection # spacy # thinc -pygments==2.11.2 - # via - # ipython - # nbconvert -pyparsing==3.0.7 - # via - # matplotlib - # packaging -pypdf2==1.27.5 - # via -r requirements.in -pyrsistent==0.18.1 +pygments==2.13.0 + # via ipython +pyparsing==3.0.9 + # via matplotlib +pypdf2==2.11.2 + # via sec-certs (./../pyproject.toml) +pyrsistent==0.19.2 # via jsonschema pysankeybeta==1.4.0 - # via -r requirements.in + # via sec-certs (./../pyproject.toml) python-dateutil==2.8.2 # via - # -r requirements.in # jupyter-client # matplotlib # pandas -pytz==2022.1 + # sec-certs (./../pyproject.toml) +pytz==2022.6 # via pandas pyyaml==6.0 - # via -r requirements.in -pyzmq==22.3.0 + # via sec-certs (./../pyproject.toml) +pyzmq==24.0.1 # via + # ipykernel # jupyter-client - # notebook -rapidfuzz==2.0.7 - # via -r requirements.in -requests==2.27.1 +rapidfuzz==2.13.3 + # via sec-certs (./../pyproject.toml) +requests==2.28.1 # via - # -r requirements.in + # sec-certs (./../pyproject.toml) # spacy -scikit-learn==1.0.2 - # via -r requirements.in +scikit-learn==1.2.0 + # via sec-certs (./../pyproject.toml) scipy==1.9.3 # via - # -r requirements.in # scikit-learn - # seaborn -seaborn==0.11.2 + # sec-certs (./../pyproject.toml) +seaborn==0.12.1 # via - # -r requirements.in # pysankeybeta -send2trash==1.8.0 - # via notebook -setuptools-scm==6.4.2 - # via -r requirements.in + # sec-certs (./../pyproject.toml) +setuptools-scm==7.0.5 + # via sec-certs (./../pyproject.toml) six==1.16.0 # via # asttokens - # bleach # html5lib # python-dateutil -smart-open==5.2.1 +smart-open==6.2.0 # via pathy -soupsieve==2.3.1 +soupsieve==2.3.2.post1 # via beautifulsoup4 spacy==3.4.3 # via - # -r requirements.in # en-core-web-sm + # sec-certs (./../pyproject.toml) spacy-legacy==3.0.10 # via spacy -spacy-loggers==1.0.2 +spacy-loggers==1.0.4 # via spacy -srsly==2.4.3 +srsly==2.4.5 # via # confection # spacy # thinc -stack-data==0.2.0 +stack-data==0.6.2 # via ipython -tabula-py==2.3.0 - # via -r requirements.in -terminado==0.13.3 - # via notebook +tabula-py==2.6.0 + # via sec-certs (./../pyproject.toml) thinc==8.1.5 # via spacy threadpoolctl==3.1.0 # via scikit-learn -tinycss2==1.1.1 - # via nbconvert tomli==2.0.1 # via setuptools-scm -tornado==6.1 +tornado==6.2 # via # ipykernel # jupyter-client - # notebook - # terminado -tqdm==4.64.0 +tqdm==4.64.1 # via - # -r requirements.in + # sec-certs (./../pyproject.toml) # spacy -traitlets==5.1.1 +traitlets==5.6.0 # via + # comm # ipykernel # ipython # ipywidgets # jupyter-client # jupyter-core # matplotlib-inline - # nbclient - # nbconvert - # nbformat - # notebook -typer==0.4.1 +typer==0.7.0 # via # pathy # spacy -typing-extensions==4.2.0 - # via pydantic -urllib3==1.26.9 +typing-extensions==4.4.0 + # via + # pydantic + # pypdf2 + # setuptools-scm +urllib3==1.26.13 # via requests -wasabi==0.9.1 +wasabi==0.10.1 # via # spacy - # spacy-loggers # thinc wcwidth==0.2.5 # via prompt-toolkit webencodings==0.5.1 - # via - # bleach - # html5lib - # tinycss2 -widgetsnbextension==3.6.0 + # via html5lib +widgetsnbextension==4.0.4 # via ipywidgets +zipp==3.11.0 + # via importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/test_requirements.in b/requirements/test_requirements.in deleted file mode 100644 index 0c69b775..00000000 --- a/requirements/test_requirements.in +++ /dev/null @@ -1,3 +0,0 @@ -pytest -coverage -pytest-cov diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index b04802c7..f60c03ca 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -1,26 +1,325 @@ -attrs==21.4.0 - # via pytest -coverage[toml]==6.3.2 +appnope==0.1.3 + # via + # ipykernel + # ipython +asttokens==2.2.1 + # via stack-data +attrs==22.1.0 + # via + # jsonschema + # pytest +backcall==0.2.0 + # via ipython +beautifulsoup4==4.11.1 + # via sec-certs (./../pyproject.toml) +billiard==4.0.2 + # via sec-certs (./../pyproject.toml) +blis==0.7.9 + # via thinc +catalogue==2.0.8 + # via + # spacy + # srsly + # thinc +certifi==2022.12.7 + # via requests +charset-normalizer==2.1.1 + # via requests +click==8.1.3 + # via + # sec-certs (./../pyproject.toml) + # typer +comm==0.1.2 + # via ipykernel +confection==0.0.3 + # via thinc +contourpy==1.0.6 + # via matplotlib +coverage[toml]==6.5.0 # via - # -r test_requirements.in # pytest-cov -iniconfig==1.1.1 + # sec-certs (./../pyproject.toml) +cycler==0.11.0 + # via matplotlib +cymem==2.0.7 + # via + # preshed + # spacy + # thinc +debugpy==1.6.4 + # via ipykernel +decorator==5.1.1 + # via ipython +deprecation==2.1.0 + # via pikepdf +distro==1.8.0 + # via tabula-py +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1.tar.gz + # via sec-certs (./../pyproject.toml) +entrypoints==0.4 + # via jupyter-client +exceptiongroup==1.0.4 # via pytest -packaging==21.3 +executing==1.2.0 + # via stack-data +fonttools==4.38.0 + # via matplotlib +html5lib==1.1 + # via sec-certs (./../pyproject.toml) +idna==3.4 + # via requests +importlib-resources==5.10.1 + # via jsonschema +iniconfig==1.1.1 # via pytest +ipykernel==6.19.1 + # via + # ipywidgets + # sec-certs (./../pyproject.toml) +ipython==8.7.0 + # via + # ipykernel + # ipywidgets +ipywidgets==8.0.3 + # via sec-certs (./../pyproject.toml) +jedi==0.18.2 + # via ipython +jinja2==3.1.2 + # via spacy +joblib==1.2.0 + # via scikit-learn +jsonschema==4.17.3 + # via sec-certs (./../pyproject.toml) +jupyter-client==7.4.8 + # via ipykernel +jupyter-core==5.1.0 + # via jupyter-client +jupyterlab-widgets==3.0.4 + # via ipywidgets +kiwisolver==1.4.4 + # via matplotlib +langcodes==3.3.0 + # via spacy +lxml==4.9.1 + # via + # pikepdf + # sec-certs (./../pyproject.toml) +markupsafe==2.1.1 + # via jinja2 +matplotlib==3.6.2 + # via + # pysankeybeta + # seaborn + # sec-certs (./../pyproject.toml) +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +murmurhash==1.0.9 + # via + # preshed + # spacy + # thinc +nest-asyncio==1.5.6 + # via + # ipykernel + # jupyter-client +networkx==2.8.8 + # via sec-certs (./../pyproject.toml) +numpy==1.23.5 + # via + # blis + # contourpy + # matplotlib + # pandas + # pysankeybeta + # scikit-learn + # scipy + # seaborn + # sec-certs (./../pyproject.toml) + # spacy + # tabula-py + # thinc +packaging==22.0 + # via + # deprecation + # ipykernel + # matplotlib + # pikepdf + # pytest + # setuptools-scm + # spacy +pandas==1.5.2 + # via + # pysankeybeta + # seaborn + # sec-certs (./../pyproject.toml) + # tabula-py +parso==0.8.3 + # via jedi +pathy==0.10.1 + # via spacy +pdftotext==2.2.2 + # via sec-certs (./../pyproject.toml) +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pikepdf==6.2.5 + # via sec-certs (./../pyproject.toml) +pillow==9.3.0 + # via + # matplotlib + # pikepdf + # sec-certs (./../pyproject.toml) +pkgconfig==1.5.5 + # via sec-certs (./../pyproject.toml) +pkgutil-resolve-name==1.3.10 + # via jsonschema +platformdirs==2.6.0 + # via jupyter-core pluggy==1.0.0 # via pytest -py==1.11.0 - # via pytest -pyparsing==3.0.7 - # via packaging -pytest==7.1.1 +preshed==3.0.8 + # via + # spacy + # thinc +prompt-toolkit==3.0.36 + # via ipython +psutil==5.9.4 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pydantic==1.10.2 + # via + # confection + # spacy + # thinc +pygments==2.13.0 + # via ipython +pyparsing==3.0.9 + # via matplotlib +pypdf2==2.11.2 + # via sec-certs (./../pyproject.toml) +pyrsistent==0.19.2 + # via jsonschema +pysankeybeta==1.4.0 + # via sec-certs (./../pyproject.toml) +pytest==7.2.0 # via - # -r test_requirements.in # pytest-cov -pytest-cov==3.0.0 - # via -r test_requirements.in + # sec-certs (./../pyproject.toml) +pytest-cov==4.0.0 + # via sec-certs (./../pyproject.toml) +python-dateutil==2.8.2 + # via + # jupyter-client + # matplotlib + # pandas + # sec-certs (./../pyproject.toml) +pytz==2022.6 + # via pandas +pyyaml==6.0 + # via sec-certs (./../pyproject.toml) +pyzmq==24.0.1 + # via + # ipykernel + # jupyter-client +rapidfuzz==2.13.3 + # via sec-certs (./../pyproject.toml) +requests==2.28.1 + # via + # sec-certs (./../pyproject.toml) + # spacy +scikit-learn==1.2.0 + # via sec-certs (./../pyproject.toml) +scipy==1.9.3 + # via + # scikit-learn + # sec-certs (./../pyproject.toml) +seaborn==0.12.1 + # via + # pysankeybeta + # sec-certs (./../pyproject.toml) +setuptools-scm==7.0.5 + # via sec-certs (./../pyproject.toml) +six==1.16.0 + # via + # asttokens + # html5lib + # python-dateutil +smart-open==6.2.0 + # via pathy +soupsieve==2.3.2.post1 + # via beautifulsoup4 +spacy==3.4.3 + # via + # en-core-web-sm + # sec-certs (./../pyproject.toml) +spacy-legacy==3.0.10 + # via spacy +spacy-loggers==1.0.4 + # via spacy +srsly==2.4.5 + # via + # confection + # spacy + # thinc +stack-data==0.6.2 + # via ipython +tabula-py==2.6.0 + # via sec-certs (./../pyproject.toml) +thinc==8.1.5 + # via spacy +threadpoolctl==3.1.0 + # via scikit-learn tomli==2.0.1 # via # coverage # pytest + # setuptools-scm +tornado==6.2 + # via + # ipykernel + # jupyter-client +tqdm==4.64.1 + # via + # sec-certs (./../pyproject.toml) + # spacy +traitlets==5.6.0 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline +typer==0.7.0 + # via + # pathy + # spacy +typing-extensions==4.4.0 + # via + # pydantic + # pypdf2 + # setuptools-scm +urllib3==1.26.13 + # via requests +wasabi==0.10.1 + # via + # spacy + # thinc +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via html5lib +widgetsnbextension==4.0.4 + # via ipywidgets +zipp==3.11.0 + # via importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/sec_certs/__init__.py b/sec_certs/__init__.py deleted file mode 100644 index 816ea2b8..00000000 --- a/sec_certs/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Tool for analysis of security certificates and their security targets (Common Criteria, NIST FIPS140-2...). -Contains three main sub-packages: -- dataset - package that holds the respective datasets and performs all processing of them -- sample - package that holds a single sample (e.g., Common Criteria certificate - CommonCriteriaCert). Mostly data structure, but can provide basic functionality. -- model - package that provides data pipelines (transformers, classifiers, ...) for complex transformations of datasets. -""" diff --git a/sec_certs/__main__.py b/sec_certs/__main__.py deleted file mode 100644 index 1d699b85..00000000 --- a/sec_certs/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys - -from sec_certs.cli import main - -if __name__ == "__main__": - sys.exit(main()) diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py deleted file mode 100644 index 145566f7..00000000 --- a/sec_certs/cert_rules.py +++ /dev/null @@ -1,248 +0,0 @@ -from __future__ import annotations - -import re -from pathlib import Path -from typing import Final - -import yaml - -# This ignores ACM and AMA SARs that are present in CC version 2 -SARS_IMPLIED_FROM_EAL: dict[str, set[tuple[str, int]]] = { - "EAL1": { - ("ADV_FSP", 1), - ("AGD_OPE", 1), - ("AGD_PRE", 1), - ("ALC_CMC", 1), - ("ALC_CMS", 1), - ("ASE_CCL", 1), - ("ASE_ECD", 1), - ("ASE_INT", 1), - ("ASE_OBJ", 1), - ("ASE_REQ", 1), - ("ASE_TSS", 1), - ("ATE_IND", 1), - ("AVA_VAN", 1), - }, - "EAL2": { - ("ADV_ARC", 1), - ("ADV_TDS", 1), - ("AGD_OPE", 1), - ("AGD_PRE", 1), - ("ALC_CMC", 2), - ("ALC_CMS", 2), - ("ALC_DEL", 1), - ("ASE_CCL", 1), - ("ASE_ECD", 1), - ("ASE_INT", 1), - ("ASE_OBJ", 2), - ("ASE_REQ", 2), - ("ASE_SPD", 1), - ("ASE_TSS", 1), - ("ATE_COV", 1), - ("ATE_FUN", 1), - ("ATE_IND", 2), - ("AVA_VAN", 2), - }, - "EAL3": { - ("ADV_ARC", 1), - ("ADV_FSP", 3), - ("ADV_TDS", 2), - ("AGD_PRE", 1), - ("ALC_CMC", 3), - ("ALC_CMS", 3), - ("ALC_DEL", 1), - ("ALC_DVS", 1), - ("ALC_LCD", 1), - ("ASE_CCL", 1), - ("ASE_ECD", 1), - ("ASE_INT", 1), - ("ASE_OBJ", 2), - ("ASE_REQ", 2), - ("ASE_SPD", 1), - ("ASE_TSS", 1), - ("ATE_COV", 2), - ("ATE_DPT", 1), - ("ATE_FUN", 1), - ("ATE_IND", 2), - ("AVA_VAN", 2), - }, - "EAL4": { - ("ADV_ARC", 1), - ("ADV_FSP", 4), - ("ADV_IMP", 1), - ("ADV_TDS", 3), - ("AGD_OPE", 1), - ("AGD_PRE", 1), - ("ALC_CMC", 4), - ("ALC_CMS", 4), - ("ALC_DEL", 1), - ("ALC_DVS", 1), - ("ALC_LCD", 1), - ("ALC_TAT", 1), - ("ASE_CCL", 1), - ("ASE_ECD", 1), - ("ASE_INT", 1), - ("ASE_OBJ", 2), - ("ASE_REQ", 2), - ("ASE_SPD", 1), - ("ASE_TSS", 1), - ("ATE_COV", 2), - ("ATE_DPT", 1), - ("ATE_FUN", 1), - ("ATE_IND", 2), - ("AVA_VAN", 3), - }, - "EAL5": { - ("ADV_ARC", 1), - ("ADV_FSP", 5), - ("ADV_IMP", 1), - ("ADV_INT", 2), - ("ADV_TDS", 4), - ("AGD_OPE", 1), - ("AGD_PRE", 1), - ("ALC_CMC", 4), - ("ALC_CMS", 5), - ("ALC_DEL", 1), - ("ALC_DVS", 1), - ("ALC_LCD", 1), - ("ALC_TAT", 2), - ("ASE_CCL", 1), - ("ASE_ECD", 1), - ("ASE_INT", 1), - ("ASE_OBJ", 2), - ("ASE_REQ", 2), - ("ASE_SPD", 1), - ("ASE_TSS", 1), - ("ATE_COV", 2), - ("ATE_DPT", 3), - ("ATE_FUN", 1), - ("ATE_IND", 2), - ("AVA_VAN", 4), - }, - "EAL6": { - ("ADV_ARC", 1), - ("ADV_FSP", 5), - ("ADV_IMP", 2), - ("ADV_INT", 3), - ("ADV_SPM", 1), - ("ADV_TDS", 5), - ("AGD_OPE", 1), - ("AGD_PRE", 1), - ("ALC_CMC", 5), - ("ALC_CMS", 5), - ("ALC_DEL", 1), - ("ALC_DVS", 2), - ("ALC_LCD", 1), - ("ALC_TAT", 3), - ("ASE_CCL", 1), - ("ASE_ECD", 1), - ("ASE_INT", 1), - ("ASE_OBJ", 2), - ("ASE_REQ", 2), - ("ASE_SPD", 1), - ("ASE_TSS", 1), - ("ATE_COV", 3), - ("ATE_DPT", 3), - ("ATE_FUN", 2), - ("ATE_IND", 2), - ("AVA_VAN", 5), - }, - "EAL7": { - ("ADV_ARC", 1), - ("ADV_FSP", 6), - ("ADV_IMP", 2), - ("ADV_INT", 3), - ("ADV_SPM", 1), - ("ADV_TDS", 6), - ("AGD_OPE", 1), - ("AGD_PRE", 1), - ("ALC_CMC", 5), - ("ALC_CMS", 5), - ("ALC_DEL", 1), - ("ALC_DVS", 2), - ("ALC_LCD", 2), - ("ALC_TAT", 3), - ("ASE_CCL", 1), - ("ASE_ECD", 1), - ("ASE_INT", 1), - ("ASE_OBJ", 2), - ("ASE_REQ", 2), - ("ASE_SPD", 1), - ("ASE_TSS", 1), - ("ATE_COV", 3), - ("ATE_DPT", 4), - ("ATE_FUN", 2), - ("ATE_IND", 3), - ("AVA_VAN", 5), - }, -} - -security_level_csv_scan = r"EAL[1-7]\+?" - - -REGEXEC_SEP = "[ ,;\\[\\]”\"')(.]" -MATCH_START = "(?P" -MATCH_END = ")" -REGEXEC_SEP_START = f"(?:^|{REGEXEC_SEP})" -REGEXEC_SEP_END = f"(?:$|{REGEXEC_SEP})" - - -SERVICE_PACK_RE = re.compile(r"(?:sp|service pack)\s{0,1}\d{1,2}", re.IGNORECASE) -RELEASE_RE = re.compile(r"(?:r|release)\s{0,1}\d{1,2}", re.IGNORECASE) -PLATFORM_REGEXES = { - "linux": re.compile(r"linux", re.IGNORECASE), - "mac_os": re.compile(r"mac\s?os\s?x?", re.IGNORECASE), - "windows": re.compile(r"windows", re.IGNORECASE), - "android": re.compile(r"android", re.IGNORECASE), - "ios": re.compile(r"(ios|iphone os)", re.IGNORECASE), -} - -FIPS_ALGS_IN_TABLE = r"(?:#[CcAa]?\s?|(?:Cert)\.?[^. ]*?\s?)(?:[CcAa]\s)?(?P\d+)" -FIPS_LIST_OF_TABLES = re.compile(r"^(?:(?:[Tt]able\s|[Ll]ist\s)(?:[Oo]f\s))[Tt]ables[\s\S]+?\f", re.MULTILINE) - - -def _load(): - script_dir = Path(__file__).parent - filepath = script_dir / "rules.yaml" - with Path(filepath).open("r") as file: - loaded = yaml.load(file, Loader=yaml.FullLoader) - return loaded - - -def _process(obj): - if isinstance(obj, dict): - return {k: _process(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [ - re.compile( - REGEXEC_SEP_START + MATCH_START + rule + MATCH_END + REGEXEC_SEP_END, - re.MULTILINE, - ) - for rule in obj - ] - - -rules = _load() - -cc_rules = {} -for rule_group in rules["cc_rules"]: - cc_rules[rule_group] = _process(rules[rule_group]) - -fips_rules = {} -for rule_group in rules["fips_rules"]: - fips_rules[rule_group] = _process(rules[rule_group]) - - -PANDAS_KEYWORDS_CATEGORIES: Final[list[str]] = [ - "symmetric_crypto", - "asymmetric_crypto", - "pq_crypto", - "hash_function", - "crypto_scheme", - "crypto_protocol", - "randomness", - "cipher_mode", - "ecc_curve", - "crypto_engine", - "crypto_library", -] diff --git a/sec_certs/cli.py b/sec_certs/cli.py deleted file mode 100644 index a25b55ee..00000000 --- a/sec_certs/cli.py +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import logging -import sys -from dataclasses import dataclass, field -from datetime import datetime -from pathlib import Path -from typing import Callable - -import click - -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.dataset import CCDataset, FIPSDataset -from sec_certs.utils.helpers import warn_if_missing_poppler, warn_if_missing_tesseract - -logger = logging.getLogger(__name__) - -EXIT_CODE_NOK: int = 1 -EXIT_CODE_OK: int = 0 - - -@dataclass -class ProcessingStep: - name: str - processing_function_name: str - preconditions: list[str] = field(default_factory=list) - precondition_error_msg: str | None = field(default=None) - pre_callback_func: Callable | None = field(default=None) - - def run(self, dset: CCDataset | FIPSDataset) -> None: - for condition in self.preconditions: - if not getattr(dset.state, condition): - err_msg = ( - self.precondition_error_msg - if self.precondition_error_msg - else f"Error, precondition to run {self.name} not met, exiting." - ) - click.echo(err_msg, err=True) - sys.exit(EXIT_CODE_NOK) - if self.pre_callback_func: - self.pre_callback_func() - - getattr(dset, self.processing_function_name)() - - -def warn_missing_libs(): - warn_if_missing_poppler() - warn_if_missing_tesseract() - - -def build_or_load_dataset( - framework: str, - inputpath: Path | None, - to_build: bool, - outputpath: Path = constants.DUMMY_NONEXISTING_PATH, -) -> CCDataset | FIPSDataset: - constructor: type[CCDataset] | type[FIPSDataset] = CCDataset if framework == "cc" else FIPSDataset - dset: CCDataset | FIPSDataset - - if to_build: - if inputpath: - print( - f"Warning: you wanted to build a dataset but you provided one in JSON -- that will be ignored. New one will be constructed at: {outputpath}" - ) - dset = constructor( - certs={}, - root_dir=outputpath, - name=framework + "_dataset", - description=f"Full {framework} dataset snapshot {datetime.now().date()}", - ) - dset.get_certs_from_web() - else: - if inputpath: - dset = constructor.from_json(inputpath) - if outputpath and dset.root_dir != outputpath: - print( - "Warning: you provided both input and output paths. The dataset from input path will get copied to output path." - ) - dset.copy_dataset(outputpath) - else: - click.echo( - "Error: If you do not use 'build' action, you must provide --input parameter to point to an existing dataset.", - err=True, - ) - sys.exit(EXIT_CODE_NOK) - - return dset - - -@click.command() -@click.argument( - "framework", - required=True, - nargs=1, - type=click.Choice(["cc", "fips"], case_sensitive=False), -) -@click.argument( - "actions", - required=True, - nargs=-1, - type=click.Choice(["all", "build", "process-aux-dsets", "download", "convert", "analyze"], case_sensitive=False), -) -@click.option( - "-o", - "--output", - type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), - help="Path where the output of the experiment will be stored. May overwrite existing content.", - default=Path("./dataset/"), - show_default=True, -) -@click.option( - "-c", - "--config", - "configpath", - default=None, - type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), - help="Path to your own config yaml file that will override the default one.", -) -@click.option( - "-i", - "--input", - "inputpath", - type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), - help="If set, the actions will be performed on a CC dataset loaded from JSON from the input path.", -) -@click.option("-q", "--quiet", is_flag=True, help="If set, will not print to stdout") -def main( - framework: str, - actions: list[str], - outputpath: Path, - configpath: str | None, - inputpath: Path | None, - silent: bool, -): - try: - file_handler = logging.FileHandler(config.log_filepath) - stream_handler = logging.StreamHandler(sys.stderr) - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - file_handler.setFormatter(formatter) - stream_handler.setFormatter(formatter) - handlers: list[logging.StreamHandler] = [file_handler] if silent else [file_handler, stream_handler] - logging.basicConfig(level=logging.INFO, handlers=handlers) - start = datetime.now() - - if configpath: - try: - config.load(Path(configpath)) - except FileNotFoundError: - click.echo("Error: Bad path to configuration file", err=True) - sys.exit(EXIT_CODE_NOK) - except ValueError as e: - click.echo(f"Error: Bad format of configuration file: {e}", err=True) - sys.exit(EXIT_CODE_NOK) - - actions_set = ( - {"build", "process-aux-dsets", "download", "convert", "analyze", "maintenances"} - if "all" in actions - else set(actions) - ) - - dset = build_or_load_dataset(framework, inputpath, "build" in actions_set, outputpath) - aux_dsets_to_handle = "PP, Maintenance updates" if framework == "cc" else "Algorithms" - aux_dsets_to_handle += "CPE, CVE" - analysis_pre_callback = None - - steps = [ - ProcessingStep( - "process-aux-dsets", - "process_auxillary_datasets", - preconditions=["meta_sources_parsed"], - precondition_error_msg=f"Error: You want to process the auxillary datasets: {aux_dsets_to_handle} , but the data from cert. framework website was not parsed. You must use 'build' action first.", - pre_callback_func=None, - ), - ProcessingStep( - "download", - "download_all_artifacts", - preconditions=["meta_sources_parsed"], - precondition_error_msg="Error: You want to download all artifacts, but the data from the cert. framework website was not parsed. You must use 'build' action first.", - pre_callback_func=None, - ), - ProcessingStep( - "convert", - "convert_all_pdfs", - preconditions=["pdfs_downloaded"], - precondition_error_msg="Error: You want to convert pdfs -> txt, but the pdfs were not downloaded. You must use 'download' action first.", - pre_callback_func=warn_missing_libs, - ), - ProcessingStep( - "analyze", - "analyze_certificates", - preconditions=["pdfs_converted", "auxillary_datasets_processed"], - precondition_error_msg="Error: You want to process txt documents of certificates, but pdfs were not converted. You must use 'convert' action first.", - pre_callback_func=analysis_pre_callback, - ), - ] - - processing_step: ProcessingStep - for processing_step in [x for x in steps if x in actions_set]: - processing_step.run(dset) - - end = datetime.now() - logger.info(f"The computation took {(end-start)} seconds.") - except Exception: - return EXIT_CODE_NOK - return EXIT_CODE_OK - - -if __name__ == "__main__": - main() diff --git a/sec_certs/config/__init__.py b/sec_certs/config/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sec_certs/config/configuration.py b/sec_certs/config/configuration.py deleted file mode 100644 index 43cfe698..00000000 --- a/sec_certs/config/configuration.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import jsonschema -import yaml - - -class Configuration: - def load(self, filepath: str | Path) -> None: - with Path(filepath).open("r") as file: - state = yaml.load(file, Loader=yaml.FullLoader) - - script_dir = Path(__file__).parent - - with (Path(script_dir) / "settings-schema.json").open("r") as file: - schema = json.loads(file.read()) - - try: - jsonschema.validate(state, schema) - except jsonschema.exceptions.ValidationError as e: - print(f"{e}\n\nIn file {filepath}") - - for k, v in state.items(): - setattr(self, k, v) - - def __getattribute__(self, key: str) -> Any: - res = object.__getattribute__(self, key) - if isinstance(res, dict) and "value" in res: - return res["value"] - return object.__getattribute__(self, key) - - def get_desription(self, key: str) -> str | None: - res = object.__getattribute__(self, key) - if isinstance(res, dict) and "description" in res: - return res["description"] - return None - - -DEFAULT_CONFIG_PATH = Path(__file__).parent / "settings.yaml" -config = Configuration() -config.load(DEFAULT_CONFIG_PATH) diff --git a/sec_certs/config/settings-schema.json b/sec_certs/config/settings-schema.json deleted file mode 100644 index a31f6b3f..00000000 --- a/sec_certs/config/settings-schema.json +++ /dev/null @@ -1,182 +0,0 @@ -{ - "title": "settings for sec-certs", - "type": "object", - "definitions": { - "settings_string_entry": { - "required": [ - "description", - "value" - ], - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "settings_boolean_entry": { - "type": "object", - "required": [ - "description", - "value" - ], - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "boolean" - } - } - }, - "settings_number_entry": { - "type": "object", - "required": [ - "description", - "value" - ], - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "number" - } - } - }, - "settings_url_entry": { - "type": "object", - "required": [ - "description", - "value" - ], - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "string", - "format": "uri", - "pattern": "^(https?|http?)://", - "minLength": 1, - "maxLength": 255 - } - } - } - }, - "properties": { - "log_filepath": { - "$ref": "#/definitions/settings_string_entry" - }, - "always_false_positive_fips_cert_id_threshold": { - "$ref": "#/definitions/settings_number_entry" - }, - "year_difference_between_validations": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 0 - } - } - } - ] - }, - "n_threads": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 1 - } - } - } - ] - }, - "cpe_n_matching_threshold": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 0, - "maximum": 100 - } - } - } - ] - }, - "cpe_n_max_matches": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "exclusiveMinimum": 0 - } - } - } - ] - }, - "cc_latest_snapshot": { - "$ref": "#/definitions/settings_url_entry" - }, - "cc_maintenances_latest_snapshot": { - "$ref": "#/definitions/settings_url_entry" - }, - "pp_latest_snapshot": { - "$ref": "#/definitions/settings_url_entry" - }, - "ignore_first_page": { - "$ref": "#/definitions/settings_boolean_entry" - }, - "cert_threshold": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 0 - } - } - } - ] - }, - "fips_latest_snapshot": { - "$ref": "#/definitions/settings_url_entry" - }, - "enable_progress_bars": { - "$ref": "#/definitions/settings_boolean_entry" - } - }, - "required": [ - "log_filepath", - "always_false_positive_fips_cert_id_threshold", - "year_difference_between_validations", - "n_threads", - "cpe_matching_threshold", - "cpe_n_max_matches", - "cc_latest_snapshot", - "cc_maintenances_latest_snapshot", - "pp_latest_snapshot", - "ignore_first_page", - "cert_threshold", - "fips_latest_snapshot", - "enable_progress_bars" - ] -} \ No newline at end of file diff --git a/sec_certs/config/settings.yaml b/sec_certs/config/settings.yaml deleted file mode 100644 index 8d645758..00000000 --- a/sec_certs/config/settings.yaml +++ /dev/null @@ -1,59 +0,0 @@ ---- -log_filepath: - description: Path to the file, relative to working directory, where the log will be stored - value: ./cert_processing_log.txt -always_false_positive_fips_cert_id_threshold: - description: - During validation we don't connect certificates with number lower than - _this_ to connections due to these numbers being typically false positives - value: 40 -year_difference_between_validations: - description: - During validation we don't connect certificates with validation dates - difference higher than _this_ - value: 7 -n_threads: - description: How many threads to use for parallel computations - value: 8 -cpe_matching_threshold: - description: Level of required string similarity between CPE and certificate name on CC CPE matching, 0-100. Lower values yield more false negatives, higher values more false positives - value: 92 -cpe_n_max_matches: - description: Maximum number of candidate CPE items that may be related to given certificate, >0 - value: 99 -cc_latest_snapshot: - description: URL from where to fetch the latest snapshot of fully processed CC dataset - value: https://seccerts.org/cc/dataset.json -cc_maintenances_latest_snapshot: - description: URL from where to fetch the latest snapshot of CC maintenance updates - value: https://seccerts.org/cc/maintenance_updates.json -pp_latest_snapshot: - description: URL from where to fetch the latest snapshot of the PP dataset - value: https://seccerts.org/static/pp.json -ignore_first_page: - description: During keyword search, first page usually contains addresses - ignore it. - value: true -cert_threshold: - description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty. - value: 5 -fips_latest_snapshot: - description: URL for the latest snapshot of FIPS dataset - value: https://seccerts.org/fips/dataset.json -fips_iut_dataset: - description: URL for the dataset of FIPS IUT data - value: https://seccerts.org/fips/iut/dataset.json -fips_iut_latest_snapshot: - description: URL for the latest snapshot of FIPS IUT data - value: https://seccerts.org/fips/iut/latest.json -fips_mip_dataset: - description: URL for the dataset of FIPS MIP data - value: https://seccerts.org/fips/mip/dataset.json -fips_mip_latest_snapshot: - description: URL for the latest snapshot of FIPS MIP data - value: https://seccerts.org/fips/mip/latest.json -minimal_token_length: - description: Minimal length of a string that will be considered as a token during keyword extraction in CVE matching - value: 3 -enable_progress_bars: - description: Whether to enable pretty-printed progress bars while processing. - value: true diff --git a/sec_certs/constants.py b/sec_certs/constants.py deleted file mode 100644 index f71bb3c3..00000000 --- a/sec_certs/constants.py +++ /dev/null @@ -1,121 +0,0 @@ -import re -from pathlib import Path - -DUMMY_NONEXISTING_PATH = Path("/this/is/dummy/nonexisting/path") - -RESPONSE_OK = 200 -RETURNCODE_OK = "ok" -RETURNCODE_NOK = "nok" -REQUEST_TIMEOUT = 10 - -MIN_CORRECT_CERT_SIZE = 5000 - -MIN_FIPS_HTML_SIZE = 64000 - -MIN_CC_HTML_SIZE = 5000000 -MIN_CC_CSV_SIZE = 700000 -MIN_CC_PP_DATASET_SIZE = 2500000 - -CPE_VERSION_NA = "-" - -RELEASE_CANDIDATE_REGEX: re.Pattern = re.compile(r"rc\d{0,2}$", re.IGNORECASE) - -FIPS_BASE_URL = "https://csrc.nist.gov" -FIPS_CMVP_URL = FIPS_BASE_URL + "/projects/cryptographic-module-validation-program" -FIPS_CAVP_URL = FIPS_BASE_URL + "/projects/Cryptographic-Algorithm-Validation-Program" -FIPS_MODULE_URL = FIPS_CMVP_URL + "/certificate/{}" -FIPS_ALG_SEARCH_URL = FIPS_CAVP_URL + "/validation-search?searchMode=implementation&page=" -FIPS_SP_URL = "https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{}.pdf" -FIPS_ACTIVE_MODULES_URL = ( - FIPS_CMVP_URL + "/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0" -) -FIPS_HISTORICAL_MODULES_URL = ( - FIPS_CMVP_URL + "/validated-modules/search?SearchMode=Advanced&CertificateStatus=Historical&ValidationYear=0" -) -FIPS_REVOKED_MODULES_URL = ( - FIPS_CMVP_URL + "/validated-modules/search?SearchMode=Advanced&CertificateStatus=Revoked&ValidationYear=0" -) -FIPS_ALG_URL = FIPS_CAVP_URL + "/details?source={}&number={}" -FIPS_IUT_URL = "https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/IUT-List" -FIPS_MIP_URL = ( - "https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/Modules-In-Process-List" -) - -FIPS_DOWNLOAD_DELAY = 1 - -FIPS_MIP_STATUS_RE = re.compile(r"^(?P[a-zA-Z ]+?) +\((?P\d{1,2}/\d{1,2}/\d{4})\)$") - -TAG_CERT_ID = "cert_id" -TAG_CC_SECURITY_LEVEL = "cc_security_level" -TAG_CC_VERSION = "cc_version" -TAG_CERT_LAB = "cert_lab" -TAG_CERT_ITEM = "cert_item" -TAG_CERT_ITEM_VERSION = "cert_item_version" -TAG_DEVELOPER = "developer" -TAG_REFERENCED_PROTECTION_PROFILES = "ref_protection_profiles" -TAG_HEADER_MATCH_RULES = "match_rules" - -FILE_ERRORS_STRATEGY = "surrogateescape" -MAX_ALLOWED_MATCH_LENGTH = 300 -LINE_SEPARATOR = " " - -GARBAGE_LINES_THRESHOLD = 30 -GARBAGE_SIZE_THRESHOLD = 1000 -GARBAGE_AVG_LLEN_THRESHOLD = 10 -GARBAGE_EVERY_SECOND_CHAR_THRESHOLD = 15 -GARBAGE_ALPHA_CHARS_THRESHOLD = 0.5 - -CC_AUSTRALIA_BASE_URL = "https://www.cyber.gov.au" -CC_AUSTRALIA_CERTIFIED_URL = ( - CC_AUSTRALIA_BASE_URL + "/acsc/view-all-content/programs/australian-information-security-evaluation-program" -) -CC_CANADA_CERTIFIED_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/certified-products" -CC_CANADA_INEVAL_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/products-evaluation" -CC_ANSSI_BASE_URL = "https://www.ssi.gouv.fr" -CC_ANSSI_CERTIFIED_URL = CC_ANSSI_BASE_URL + "/en/products/certified-products/" -CC_BSI_BASE_URL = "https://www.bsi.bund.de/" -CC_BSI_CERTIFIED_URL = CC_BSI_BASE_URL + "EN2021/Topics/Certification/certified_products/certified_products_node.html" -CC_INDIA_CERTIFIED_URL = "https://www.commoncriteria-india.gov.in/product-certified" -CC_INDIA_ARCHIVED_URL = "https://www.commoncriteria-india.gov.in/archived-prod-cer" -CC_ITALY_BASE_URL = "https://www.ocsi.gov.it" -CC_ITALY_CERTIFIED_URL = CC_ITALY_BASE_URL + "/index.php/elenchi-certificazioni/prodotti-certificati.html" -CC_ITALY_INEVAL_URL = CC_ITALY_BASE_URL + "/index.php/elenchi-certificazioni/in-corso-di-valutazione.html" -CC_JAPAN_BASE_URL = "https://www.ipa.go.jp/security/jisec/jisec_e" -CC_JAPAN_CERT_BASE_URL = CC_JAPAN_BASE_URL + "/certified_products" -CC_JAPAN_CERTIFIED_URL = CC_JAPAN_BASE_URL + "/certified_products/certfy_list_e31.html" -CC_JAPAN_ARCHIVED_URL = CC_JAPAN_BASE_URL + "/certified_products/certfy_list_e_archive.html" -CC_JAPAN_INEVAL_URL = CC_JAPAN_BASE_URL + "/prdct_in_eval.html" -CC_MALAYSIA_BASE_URL = "https://iscb.cybersecurity.my" -CC_MALAYSIA_CERTIFIED_URL = ( - CC_MALAYSIA_BASE_URL + "/en/index.php/certification/product-certification/mycc/certified-products-and-systems" -) -CC_MALAYSIA_INEVAL_URL = ( - CC_MALAYSIA_BASE_URL - + "/en/index.php/certification/product-certification/mycc/list-of-products-and-systems-under-evaluation-or-maintenance" -) -CC_NETHERLANDS_BASE_URL = "https://www.tuv-nederland.nl/common-criteria" -CC_NETHERLANDS_CERTIFIED_URL = CC_NETHERLANDS_BASE_URL + "/certificates.html" -CC_NETHERLANDS_INEVAL_URL = CC_NETHERLANDS_BASE_URL + "/ongoing-certifications.html" -CC_NORWAY_CERTIFIED_URL = "https://sertit.no/certified-products/category1919.html" -CC_NORWAY_ARCHIVED_URL = "https://sertit.no/certified-products/product-archive/" -CC_KOREA_EN_URL = "https://itscc.kr/main/main.do?accessMode=home_en" -CC_KOREA_CERTIFIED_URL = "https://itscc.kr/certprod/list.do" -CC_KOREA_PRODUCT_URL = "https://itscc.kr/certprod/view.do?product_id={}&product_class=1" -CC_SINGAPORE_BASE_URL = "https://www.csa.gov.sg" -CC_SINGAPORE_CERTIFIED_URL = ( - CC_SINGAPORE_BASE_URL + "/Programmes/certification-and-labelling-schemes/csa-common-criteria/product-list" -) -CC_SINGAPORE_ARCHIVED_URL = ( - CC_SINGAPORE_BASE_URL + "/Programmes/certification-and-labelling-schemes/csa-common-criteria/product-archives" -) -CC_SPAIN_BASE_URL = "https://oc.ccn.cni.es" -CC_SPAIN_CERTIFIED_URL = CC_SPAIN_BASE_URL + "/en/certified-products/certified-products" -CC_SWEDEN_BASE_URL = "https://www.fmv.se" -CC_SWEDEN_CERTIFIED_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/certifikat-utgivna-av-csec/" -CC_SWEDEN_INEVAL_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/pagaende-certifieringar/" -CC_SWEDEN_ARCHIVED_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/arkiverade-certifikat-aldre-an-5-ar/" -CC_TURKEY_ARCHIVED_URL = "https://statik.tse.org.tr/upload/tr/dosya/icerikyonetimi/3300/03112021143434-2.pdf" -CC_USA_BASE_URL = "https://www.niap-ccevs.org" -CC_USA_CERTIFIED_URL = CC_USA_BASE_URL + "/Product/PCL.cfm" -CC_USA_INEVAL_URL = CC_USA_BASE_URL + "/Product/PINE.cfm" -CC_USA_ARCHIVED_URL = CC_USA_BASE_URL + "/Product/Archived.cfm" diff --git a/sec_certs/dataset/__init__.py b/sec_certs/dataset/__init__.py deleted file mode 100644 index bace4ddd..00000000 --- a/sec_certs/dataset/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -"""This package exposes Datasets of various Samples, both primary (Common Criteria, FIPS) and auxillary (CVEs, CPEs, ...)""" - -from sec_certs.dataset.common_criteria import CCDataset, CCDatasetMaintenanceUpdates -from sec_certs.dataset.cpe import CPEDataset -from sec_certs.dataset.cve import CVEDataset -from sec_certs.dataset.fips import FIPSDataset -from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.dataset.fips_iut import IUTDataset -from sec_certs.dataset.fips_mip import MIPDataset -from sec_certs.dataset.protection_profile import ProtectionProfileDataset - -__all__ = [ - "CCDataset", - "CCDatasetMaintenanceUpdates", - "CPEDataset", - "CVEDataset", - "FIPSDataset", - "FIPSAlgorithmDataset", - "IUTDataset", - "MIPDataset", - "ProtectionProfileDataset", -] diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py deleted file mode 100644 index b4a7a21f..00000000 --- a/sec_certs/dataset/common_criteria.py +++ /dev/null @@ -1,1674 +0,0 @@ -from __future__ import annotations - -import itertools -import json -import locale -import shutil -import tempfile -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -from typing import ClassVar, Iterator - -import numpy as np -import pandas as pd -import requests -import tabula -from bs4 import BeautifulSoup, NavigableString, Tag - -import sec_certs.utils.sanitization -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.dataset.cpe import CPEDataset -from sec_certs.dataset.cve import CVEDataset -from sec_certs.dataset.dataset import AuxillaryDatasets, Dataset, logger -from sec_certs.dataset.protection_profile import ProtectionProfileDataset -from sec_certs.model.reference_finder import ReferenceFinder -from sec_certs.model.sar_transformer import SARTransformer -from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder -from sec_certs.sample.cc_certificate_id import CertificateId -from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, serialize -from sec_certs.utils import helpers as helpers -from sec_certs.utils import parallel_processing as cert_processing -from sec_certs.utils.sanitization import sanitize_navigable_string as sns - - -@dataclass -class CCAuxillaryDatasets(AuxillaryDatasets): - cpe_dset: CPEDataset | None = None - cve_dset: CVEDataset | None = None - pp_dset: ProtectionProfileDataset | None = None - mu_dset: CCDatasetMaintenanceUpdates | None = None - - -class CCDataset(Dataset[CommonCriteriaCert, CCAuxillaryDatasets], ComplexSerializableType): - """ - Class that holds CommonCriteriaCert. Serializable into json, pandas, dictionary. Conveys basic certificate manipulations - and dataset transformations. Many private methods that perform internal operations, feel free to exploit them. - """ - - def __init__( - self, - certs: dict[str, CommonCriteriaCert] = dict(), - root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH, - name: str | None = None, - description: str = None, - state: Dataset.DatasetInternalState | None = None, - auxillary_datasets: CCAuxillaryDatasets | None = None, - ): - self.certs = certs - self.timestamp = datetime.now() - self.sha256_digest = "not implemented" - self.name = name if name else type(self).__name__ + " dataset" - self.description = description if description else datetime.now().strftime("%d/%m/%Y %H:%M:%S") - self.state = state if state else self.DatasetInternalState() - - self.auxillary_datasets: CCAuxillaryDatasets = ( - auxillary_datasets if auxillary_datasets else CCAuxillaryDatasets() - ) - - self.root_dir = Path(root_dir) - - def to_pandas(self) -> pd.DataFrame: - """ - Return self serialized into pandas DataFrame - """ - df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=CommonCriteriaCert.pandas_columns) - df = df.set_index("dgst") - - df.not_valid_before = pd.to_datetime(df.not_valid_before, infer_datetime_format=True) - df.not_valid_after = pd.to_datetime(df.not_valid_after, infer_datetime_format=True) - df = df.astype( - {"category": "category", "status": "category", "scheme": "category", "cert_lab": "category"} - ).fillna(value=np.nan) - df = df.loc[ - ~df.manufacturer.isnull() - ] # Manually delete one certificate with None manufacturer (seems to have many blank fields) - - # Categorize EAL - df.eal = df.eal.fillna(value=np.nan) - df.eal = pd.Categorical(df.eal, categories=sorted(df.eal.dropna().unique().tolist()), ordered=True) - - # Introduce year when cert got valid - df["year_from"] = pd.DatetimeIndex(df.not_valid_before).year - - return df - - @property - def reports_dir(self) -> Path: - """ - Returns directory that holds files associated with certification reports - """ - return self.certs_dir / "reports" - - @property - def reports_pdf_dir(self) -> Path: - """ - Returns directory that holds PDFs associated with certification reports - """ - return self.reports_dir / "pdf" - - @property - def reports_txt_dir(self) -> Path: - """ - Returns directory that holds TXTs associated with certification reports - """ - return self.reports_dir / "txt" - - @property - def targets_dir(self) -> Path: - """ - Returns directory that holds files associated with security targets - """ - return self.certs_dir / "targets" - - @property - def targets_pdf_dir(self) -> Path: - """ - Returns directory that holds PDFs associated with security targets - """ - return self.targets_dir / "pdf" - - @property - def targets_txt_dir(self) -> Path: - """ - Returns directory that holds TXTs associated with security targets - """ - return self.targets_dir / "txt" - - @property - def pp_dataset_path(self) -> Path: - """ - Returns directory that holds files associated with Protection profiles - """ - return self.auxillary_datasets_dir / "pp_dataset.json" - - @property - def mu_dataset_dir(self) -> Path: - """ - Returns directory that holds dataset of maintenance updates - """ - return self.auxillary_datasets_dir / "maintenances" - - @property - def mu_dataset_path(self) -> Path: - """ - Returns json that holds the datase of maintenance updates - """ - return self.mu_dataset_dir / "maintenance_updates.json" - - BASE_URL: ClassVar[str] = "https://www.commoncriteriaportal.org" - - HTML_PRODUCTS_URL = { - "cc_products_active.html": BASE_URL + "/products/", - "cc_products_archived.html": BASE_URL + "/products/index.cfm?archived=1", - } - HTML_LABS_URL = {"cc_labs.html": BASE_URL + "/labs"} - CSV_PRODUCTS_URL = { - "cc_products_active.csv": BASE_URL + "/products/certified_products.csv", - "cc_products_archived.csv": BASE_URL + "/products/certified_products-archived.csv", - } - PP_URL = { - "cc_pp_active.html": BASE_URL + "/pps/", - "cc_pp_collaborative.html": BASE_URL + "/pps/collaborativePP.cfm?cpp=1", - "cc_pp_archived.html": BASE_URL + "/pps/index.cfm?archived=1", - } - PP_CSV = {"cc_pp_active.csv": BASE_URL + "/pps/pps.csv", "cc_pp_archived.csv": BASE_URL + "/pps/pps-archived.csv"} - - @property - def active_html_tuples(self) -> list[tuple[str, Path]]: - """ - Returns List Tuple[str, Path] where first element is name of html file and second element is its Path. - The files correspond to html files parsed from CC website that list all *active* certificates. - """ - return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if "active" in y] - - @property - def archived_html_tuples(self) -> list[tuple[str, Path]]: - """ - Returns List Tuple[str, Path] where first element is name of html file and second element is its Path. - The files correspond to html files parsed from CC website that list all *archived* certificates. - """ - return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if "archived" in y] - - @property - def active_csv_tuples(self) -> list[tuple[str, Path]]: - """ - Returns List Tuple[str, Path] where first element is name of csv file and second element is its Path. - The files correspond to csv files downloaded from CC website that list all *active* certificates. - """ - return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if "active" in y] - - @property - def archived_csv_tuples(self) -> list[tuple[str, Path]]: - """ - Returns List Tuple[str, Path] where first element is name of csv file and second element is its Path. - The files correspond to csv files downloaded from CC website that list all *archived* certificates. - """ - return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if "archived" in y] - - @classmethod - def from_web_latest(cls) -> CCDataset: - """ - Fetches the fresh snapshot of CCDataset from seccerts.org - """ - return cls.from_web(config.cc_latest_snapshot, "Downloading CC Dataset", "cc_latest_dataset.json") - - def _set_local_paths(self): - super()._set_local_paths() - - if self.auxillary_datasets.pp_dset: - self.auxillary_datasets.pp_dset.json_path = self.pp_dataset_path - - if self.auxillary_datasets.mu_dset: - self.auxillary_datasets.mu_dset.root_dir = self.mu_dataset_dir - - for cert in self: - cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir) - # TODO: This forgets to set local paths for other auxillary datasets - - def _merge_certs(self, certs: dict[str, CommonCriteriaCert], cert_source: str | None = None) -> None: - """ - Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates - """ - new_certs = {x.dgst: x for x in certs.values() if x not in self} - certs_to_merge = [x for x in certs.values() if x in self] - self.certs.update(new_certs) - - for crt in certs_to_merge: - self[crt.dgst].merge(crt, cert_source) - - logger.info(f"Added {len(new_certs)} new and merged further {len(certs_to_merge)} certificates to the dataset.") - - def _download_csv_html_resources(self, get_active: bool = True, get_archived: bool = True) -> None: - self.web_dir.mkdir(parents=True, exist_ok=True) - - html_items = [] - csv_items = [] - if get_active is True: - html_items.extend(self.active_html_tuples) - csv_items.extend(self.active_csv_tuples) - if get_archived is True: - html_items.extend(self.archived_html_tuples) - csv_items.extend(self.archived_csv_tuples) - - html_urls, html_paths = [x[0] for x in html_items], [x[1] for x in html_items] - csv_urls, csv_paths = [x[0] for x in csv_items], [x[1] for x in csv_items] - - logger.info("Downloading required csv and html files.") - helpers.download_parallel(html_urls, html_paths) - helpers.download_parallel(csv_urls, csv_paths) - - @serialize - def get_certs_from_web( - self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, get_archived: bool = True - ) -> None: - """ - Downloads CSV and HTML files that hold lists of certificates from common criteria website. Parses these files - and constructs CommonCriteriaCert objects, fills the dataset with those. - - :param bool to_download: If CSV and HTML files shall be downloaded (or existing files utilized), defaults to True - :param bool keep_metadata: If CSV and HTML files shall be kept on disk after download, defaults to True - :param bool get_active: If active certificates shall be parsed, defaults to True - :param bool get_archived: If archived certificates shall be parsed, defaults to True - """ - if to_download is True: - self._download_csv_html_resources(get_active, get_archived) - - logger.info("Adding CSV certificates to CommonCriteria dataset.") - csv_certs = self._get_all_certs_from_csv(get_active, get_archived) - self._merge_certs(csv_certs, cert_source="csv") - - # Someway along the way, 3 certificates get lost. - logger.info("Adding HTML certificates to CommonCriteria dataset.") - html_certs = self._get_all_certs_from_html(get_active, get_archived) - self._merge_certs(html_certs, cert_source="html") - - logger.info(f"The resulting dataset has {len(self)} certificates.") - - if not keep_metadata: - shutil.rmtree(self.web_dir) - - self._set_local_paths() - self.state.meta_sources_parsed = True - - def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> dict[str, CommonCriteriaCert]: - """ - Creates dictionary of new certificates from csv sources. - """ - csv_sources = list(self.CSV_PRODUCTS_URL.keys()) - csv_sources = [x for x in csv_sources if "active" not in x or get_active] - csv_sources = [x for x in csv_sources if "archived" not in x or get_archived] - - new_certs = {} - for file in csv_sources: - partial_certs = self._parse_single_csv(self.web_dir / file) - logger.info(f"Parsed {len(partial_certs)} certificates from: {file}") - new_certs.update(partial_certs) - return new_certs - - @staticmethod - def _parse_single_csv(file: Path) -> dict[str, CommonCriteriaCert]: - """ - Using pandas, this parses a single CSV file. - """ - - def map_ip_to_hostname(url: str) -> str: - if not url: - return url - tokens = url.split("/") - relative_path = "/" + "/".join(tokens[3:]) - return CCDataset.BASE_URL + relative_path - - def _get_primary_key_str(row: Tag): - prim_key = row["category"] + row["cert_name"] + row["report_link"] - return prim_key - - if "active" in str(file): - cert_status = "active" - else: - cert_status = "archived" - - csv_header = [ - "category", - "cert_name", - "manufacturer", - "scheme", - "security_level", - "protection_profiles", - "not_valid_before", - "not_valid_after", - "report_link", - "st_link", - "maintenance_date", - "maintenance_title", - "maintenance_report_link", - "maintenance_st_link", - ] - - # TODO: Now skipping bad lines, smarter heuristics to be built for dumb files - df = pd.read_csv(file, engine="python", encoding="windows-1252", on_bad_lines="skip") - df = df.rename(columns={x: y for (x, y) in zip(list(df.columns), csv_header)}) - - df["is_maintenance"] = ~df.maintenance_title.isnull() - df = df.fillna(value="") - - df[["not_valid_before", "not_valid_after", "maintenance_date"]] = df[ - ["not_valid_before", "not_valid_after", "maintenance_date"] - ].apply(pd.to_datetime) - - df["dgst"] = df.apply(lambda row: helpers.get_first_16_bytes_sha256(_get_primary_key_str(row)), axis=1) - - df_base = df.loc[~df.is_maintenance].copy() - df_main = df.loc[df.is_maintenance].copy() - - df_base.report_link = df_base.report_link.map(map_ip_to_hostname) - df_base.st_link = df_base.st_link.map(map_ip_to_hostname) - - df_main.maintenance_report_link = df_main.maintenance_report_link.map(map_ip_to_hostname) - df_main.maintenance_st_link = df_main.maintenance_st_link.map(map_ip_to_hostname) - - n_all = len(df_base) - n_deduplicated = len(df_base.drop_duplicates(subset=["dgst"])) - if (n_dup := n_all - n_deduplicated) > 0: - logger.warning(f"The CSV {file} contains {n_dup} duplicates by the primary key.") - - df_base = df_base.drop_duplicates(subset=["dgst"]) - df_main = df_main.drop_duplicates() - - profiles = { - x.dgst: { - ProtectionProfile(pp_name=y, pp_eal=None) - for y in sec_certs.utils.sanitization.sanitize_protection_profiles(x.protection_profiles) - } - for x in df_base.itertuples() - } - updates: dict[str, set] = {x.dgst: set() for x in df_base.itertuples()} - for x in df_main.itertuples(): - updates[x.dgst].add( - CommonCriteriaCert.MaintenanceReport( - x.maintenance_date.date(), x.maintenance_title, x.maintenance_report_link, x.maintenance_st_link - ) - ) - - certs = { - x.dgst: CommonCriteriaCert( - cert_status, - x.category, - x.cert_name, - x.manufacturer, - x.scheme, - x.security_level, - x.not_valid_before, - x.not_valid_after, - x.report_link, - x.st_link, - None, - None, - profiles.get(x.dgst, None), - updates.get(x.dgst, None), - None, - None, - None, - ) - for x in df_base.itertuples() - } - return certs - - def _get_all_certs_from_html(self, get_active: bool, get_archived: bool) -> dict[str, CommonCriteriaCert]: - """ - Prepares dictionary of certificates from all html files. - """ - html_sources = list(self.HTML_PRODUCTS_URL.keys()) - if get_active is False: - html_sources = [x for x in html_sources if "active" not in x] - if get_archived is False: - html_sources = [x for x in html_sources if "archived" not in x] - - new_certs = {} - for file in html_sources: - partial_certs = self._parse_single_html(self.web_dir / file) - logger.info(f"Parsed {len(partial_certs)} certificates from: {file}") - new_certs.update(partial_certs) - return new_certs - - @staticmethod - def _parse_single_html(file: Path) -> dict[str, CommonCriteriaCert]: - """ - Prepares a dictionary of certificates from a single html file. - """ - - def _get_timestamp_from_footer(footer): - locale.setlocale(locale.LC_ALL, "en_US") - footer_text = list(footer.stripped_strings)[0] - date_string = footer_text.split(",")[1:3] - time_string = footer_text.split(",")[3].split(" at ")[1] - formatted_datetime = date_string[0] + date_string[1] + " " + time_string - return datetime.strptime(formatted_datetime, " %B %d %Y %I:%M %p") - - def _parse_table( - soup: BeautifulSoup, cert_status: str, table_id: str, category_string: str - ) -> dict[str, CommonCriteriaCert]: - tables = soup.find_all("table", id=table_id) - - if not len(tables) <= 1: - raise ValueError( - f'The "{file.name}" was expected to contain <1 element. Instead, it contains: {len(tables)}
elements.' - ) - - if not tables: - return {} - - table = tables[0] - rows = list(table.find_all("tr")) - # header, footer = rows[0], rows[1] - body = rows[2:] - - # It's possible to obtain timestamp of the moment when the list was generated. It's identical for each table and should thus only be obtained once. Not necessarily in each table - # timestamp = _get_timestamp_from_footer(footer) - - # The following unused snippet extracts expected number of certs from the table - # caption_str = str(table.findAll('caption')) - # n_expected_certs = int(caption_str.split(category_string + ' – ')[1].split(' Certified Products')[0]) - - try: - table_certs = { - x.dgst: x - for x in [CommonCriteriaCert.from_html_row(row, cert_status, category_string) for row in body] - } - except ValueError as e: - raise ValueError(f"Bad html file: {file.name} ({str(e)})") from e - - return table_certs - - if "active" in str(file): - cert_status = "active" - else: - cert_status = "archived" - - cc_cat_abbreviations = ["AC", "BP", "DP", "DB", "DD", "IC", "KM", "MD", "MF", "NS", "OS", "OD", "DG", "TC"] - cc_table_ids = ["tbl" + x for x in cc_cat_abbreviations] - cc_categories = [ - "Access Control Devices and Systems", - "Boundary Protection Devices and Systems", - "Data Protection", - "Databases", - "Detection Devices and Systems", - "ICs, Smart Cards and Smart Card-Related Devices and Systems", - "Key Management Systems", - "Mobility", - "Multi-Function Devices", - "Network and Network-Related Devices and Systems", - "Operating Systems", - "Other Devices and Systems", - "Products for Digital Signatures", - "Trusted Computing", - ] - cat_dict = {x: y for (x, y) in zip(cc_table_ids, cc_categories)} - - with file.open("r") as handle: - soup = BeautifulSoup(handle, "html5lib") - - certs = {} - for key, val in cat_dict.items(): - certs.update(_parse_table(soup, cert_status, key, val)) - - return certs - - def _download_all_artifacts_body(self, fresh: bool = True) -> None: - self._download_reports(fresh) - self._download_targets(fresh) - - def _download_reports(self, fresh: bool = True) -> None: - self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) - certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh) and x.report_link] - - if fresh: - logger.info("Downloading PDFs of CC certification reports.") - if not fresh and certs_to_process: - logger.info( - f"Downloading {len(certs_to_process)} PDFs of CC certification reports for which previous download failed." - ) - - cert_processing.process_parallel( - CommonCriteriaCert.download_pdf_report, - certs_to_process, - config.n_threads, - progress_bar_desc="Downloading PDFs of CC certification reports", - ) - - def _download_targets(self, fresh: bool = True) -> None: - self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) - certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh)] - - if fresh: - logger.info("Downloading PDFs of CC security targets.") - if not fresh and certs_to_process: - logger.info( - f"Downloading {len(certs_to_process)} PDFs of CC security targets for which previous download failed.." - ) - - cert_processing.process_parallel( - CommonCriteriaCert.download_pdf_st, - certs_to_process, - config.n_threads, - progress_bar_desc="Downloading PDFs of CC security targets", - ) - - def _convert_reports_to_txt(self, fresh: bool = True) -> None: - self.reports_txt_dir.mkdir(parents=True, exist_ok=True) - certs_to_process = [x for x in self if x.state.report_is_ok_to_convert(fresh)] - - if fresh: - logger.info("Converting PDFs of certification reports to txt.") - if not fresh and certs_to_process: - logger.info( - f"Converting {len(certs_to_process)} PDFs of certification reports to txt for which previous conversion failed." - ) - - cert_processing.process_parallel( - CommonCriteriaCert.convert_report_pdf, - certs_to_process, - config.n_threads, - progress_bar_desc="Converting PDFs of certification reports to txt", - ) - - def _convert_targets_to_txt(self, fresh: bool = True) -> None: - self.targets_txt_dir.mkdir(parents=True, exist_ok=True) - certs_to_process = [x for x in self if x.state.st_is_ok_to_convert(fresh)] - - if fresh: - logger.info("Converting PDFs of security targets to txt.") - if not fresh and certs_to_process: - logger.info( - f"Converting {len(certs_to_process)} PDFs of security targets to txt for which previous conversion failed." - ) - - cert_processing.process_parallel( - CommonCriteriaCert.convert_st_pdf, - certs_to_process, - config.n_threads, - progress_bar_desc="Converting PDFs of security targets to txt", - ) - - def _convert_all_pdfs_body(self, fresh: bool = True) -> None: - self._convert_reports_to_txt(fresh) - self._convert_targets_to_txt(fresh) - - def _extract_report_metadata(self) -> None: - logger.info("Extracting report metadata") - certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - CommonCriteriaCert.extract_report_pdf_metadata, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting report metadata", - ) - self.update_with_certs(processed_certs) - - def _extract_targets_metadata(self) -> None: - logger.info("Extracting target metadata") - certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - CommonCriteriaCert.extract_st_pdf_metadata, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting target metadata", - ) - self.update_with_certs(processed_certs) - - def _extract_pdf_metadata(self) -> None: - self._extract_report_metadata() - self._extract_targets_metadata() - - def _extract_report_frontpage(self) -> None: - logger.info("Extracting report frontpages") - certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - CommonCriteriaCert.extract_report_pdf_frontpage, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting report frontpages", - ) - self.update_with_certs(processed_certs) - - def _extract_targets_frontpage(self) -> None: - logger.info("Extracting target frontpages") - certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - CommonCriteriaCert.extract_st_pdf_frontpage, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting target frontpages", - ) - self.update_with_certs(processed_certs) - - def _extract_pdf_frontpage(self) -> None: - self._extract_report_frontpage() - self._extract_targets_frontpage() - - def _extract_report_keywords(self) -> None: - logger.info("Extracting report keywords") - certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - CommonCriteriaCert.extract_report_pdf_keywords, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting report keywords", - ) - self.update_with_certs(processed_certs) - - def _extract_targets_keywords(self) -> None: - logger.info("Extracting target keywords") - certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - CommonCriteriaCert.extract_st_pdf_keywords, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting target keywords", - ) - self.update_with_certs(processed_certs) - - def _extract_pdf_keywords(self) -> None: - self._extract_report_keywords() - self._extract_targets_keywords() - - def extract_data(self) -> None: - logger.info("Extracting various data from certification artifacts") - self._extract_pdf_metadata() - self._extract_pdf_frontpage() - self._extract_pdf_keywords() - - def _compute_cert_labs(self) -> None: - logger.info("Computing heuristics: Deriving information about laboratories involved in certification.") - certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] - for cert in certs_to_process: - cert.compute_heuristics_cert_lab() - - def _compute_normalized_cert_ids(self) -> None: - logger.info("Computing heuristics: Deriving information about certificate ids from artifacts.") - for cert in self: - cert.compute_heuristics_cert_id() - - def _compute_transitive_vulnerabilities(self): - logger.info("omputing heuristics: computing transitive vulnerabilities in referenc(ed/ing) certificates.") - transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: cert.heuristics.cert_id) - transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.report_references) - - for dgst in self.certs: - transitive_cve = transitive_cve_finder.predict_single_cert(dgst) - - self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves - self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves - - def _compute_heuristics(self) -> None: - self._compute_normalized_cert_ids() - super()._compute_heuristics() - self._compute_cert_labs() - self._compute_sars() - - def _compute_sars(self) -> None: - logger.info("Computing heuristics: Computing SARs") - transformer = SARTransformer().fit(self.certs.values()) - for cert in self: - cert.heuristics.extracted_sars = transformer.transform_single_cert(cert) - - def _compute_references(self) -> None: - def ref_lookup(kw_attr): - def func(cert): - kws = getattr(cert.pdf_data, kw_attr) - if not kws: - return set() - res = set() - for scheme, matches in kws["cc_cert_id"].items(): - for match in matches.keys(): - try: - canonical = CertificateId(scheme, match).canonical - res.add(canonical) - except Exception: - res.add(match) - return res - - return func - - logger.info("omputing heuristics: references between certificates.") - for ref_source in ("report", "st"): - kw_source = f"{ref_source}_keywords" - dep_attr = f"{ref_source}_references" - - finder = ReferenceFinder() - finder.fit(self.certs, lambda cert: cert.heuristics.cert_id, ref_lookup(kw_source)) # type: ignore - - for dgst in self.certs: - setattr(self.certs[dgst].heuristics, dep_attr, finder.predict_single_cert(dgst, keep_unknowns=False)) - - def process_auxillary_datasets(self, download_fresh: bool = False) -> None: - """ - Processes all auxillary datasets needed during computation. On top of base-class processing, - CC handles protection profiles and maintenance updates. - """ - super().process_auxillary_datasets(download_fresh) - self.auxillary_datasets.pp_dset = self.process_protection_profiles(to_download=download_fresh) - self.auxillary_datasets.mu_dset = self.process_maintenance_updates(to_download=download_fresh) - - @serialize - def process_protection_profiles( - self, to_download: bool = True, keep_metadata: bool = True - ) -> ProtectionProfileDataset: - """ - Downloads new snapshot of dataset with processed protection profiles (if it doesn't exist) and links PPs - with certificates within self. Assigns PPs to all certificates - - :param bool to_download: If dataset should be downloaded or fetched from json, defaults to True - :param bool keep_metadata: If json related to the PP dataset should be kept on drive, defaults to True - :raises RuntimeError: When building of PPDataset fails - """ - logger.info("Processing protection profiles.") - - self.auxillary_datasets_dir.mkdir(parents=True, exist_ok=True) - - if to_download or not self.pp_dataset_path.exists(): - pp_dataset = ProtectionProfileDataset.from_web(self.pp_dataset_path) - else: - pp_dataset = ProtectionProfileDataset.from_json(self.pp_dataset_path) - - for cert in self: - if cert.protection_profiles is None: - raise RuntimeError("Building of the dataset probably failed - this should not be happening.") - cert.protection_profiles = {pp_dataset.pps.get((x.pp_name, x.pp_link), x) for x in cert.protection_profiles} - - if not keep_metadata: - self.pp_dataset_path.unlink() - - return pp_dataset - - def process_maintenance_updates(self, to_download: bool = True) -> CCDatasetMaintenanceUpdates: - """ - Downloads or loads from json a dataset of maintenance updates. Runs analysis on that dataset if it's not completed. - :return CCDatasetMaintenanceUpdates: the resulting dataset of maintenance updates - """ - - logger.info("Processing maintenace updates") - self.mu_dataset_dir.mkdir(parents=True, exist_ok=True) - - if to_download or not self.mu_dataset_path.exists(): - maintained_certs: list[CommonCriteriaCert] = [x for x in self if x.maintenance_updates] - updates = list( - itertools.chain.from_iterable( - CommonCriteriaMaintenanceUpdate.get_updates_from_cc_cert(x) for x in maintained_certs - ) - ) - update_dset = CCDatasetMaintenanceUpdates( - {x.dgst: x for x in updates}, root_dir=self.mu_dataset_dir, name="maintenance_updates" - ) - else: - update_dset = CCDatasetMaintenanceUpdates.from_json(self.mu_dataset_path) - - if not update_dset.state.artifacts_downloaded: - update_dset.download_all_artifacts() - if not update_dset.state.pdfs_converted: - update_dset.convert_all_pdfs() - if not update_dset.state.certs_analyzed: - update_dset.extract_data() - - return update_dset - - -class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType): - """ - Dataset of maintenance updates related to certificates of CCDataset dataset. - Should be used merely for actions related to Maintenance updates: download pdfs, convert pdfs, extract data from pdfs - """ - - # Quite difficult to achieve correct behaviour with MyPy here, opting for ignore - def __init__( - self, - certs: dict[str, CommonCriteriaMaintenanceUpdate] = dict(), # type: ignore - root_dir: Path = constants.DUMMY_NONEXISTING_PATH, - name: str = "dataset name", - description: str = "dataset_description", - state: CCDataset.DatasetInternalState | None = None, - ): - super().__init__(certs, root_dir, name, description, state) # type: ignore - self.state.meta_sources_parsed = True - - @property - def certs_dir(self) -> Path: - return self.root_dir - - def __iter__(self) -> Iterator[CommonCriteriaMaintenanceUpdate]: - yield from self.certs.values() # type: ignore - - def _compute_heuristics(self) -> None: - raise NotImplementedError - - def compute_related_cves(self) -> None: - raise NotImplementedError - - def process_auxillary_datasets(self, download_fresh: bool = False) -> None: - raise NotImplementedError - - def analyze_certificates(self) -> None: - raise NotImplementedError - - def get_certs_from_web( - self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, get_archived: bool = True - ) -> None: - raise NotImplementedError - - @classmethod - def from_json(cls, input_path: str | Path) -> CCDatasetMaintenanceUpdates: - input_path = Path(input_path) - with input_path.open("r") as handle: - dset = json.load(handle, cls=CustomJSONDecoder) - dset._root_dir = Path(input_path).parent - return dset - - def to_pandas(self) -> pd.DataFrame: - df = pd.DataFrame( - [x.pandas_tuple for x in self.certs.values()], columns=CommonCriteriaMaintenanceUpdate.pandas_columns - ) - df = df.set_index("dgst") - df.index.name = "dgst" - - df.maintenance_date = pd.to_datetime(df.maintenance_date, infer_datetime_format=True) - df = df.fillna(value=np.nan) - - return df - - @classmethod - def from_web_latest(cls) -> CCDatasetMaintenanceUpdates: - with tempfile.TemporaryDirectory() as tmp_dir: - dset_path = Path(tmp_dir) / "cc_maintenances_latest_dataset.json" - helpers.download_file(config.cc_maintenances_latest_snapshot, dset_path) - return cls.from_json(dset_path) - - def get_n_maintenances_df(self) -> pd.DataFrame: - """ - Returns a DataFrame with CommonCriteriaCert digest as an index, and number of registered maintenances as a value - """ - main_df = self.to_pandas() - main_df.maintenance_date = main_df.maintenance_date.dt.date - n_maintenances = ( - main_df.groupby("related_cert_digest").name.count().rename("n_maintenances").fillna(0).astype("int32") - ) - - n_maintenances.index.name = "dgst" - return n_maintenances - - def get_maintenance_dates_df(self) -> pd.DataFrame: - """ - Returns a DataFrame with CommonCriteriaCert digest as an index, and all the maintenance dates as a value. - """ - main_dates = self.to_pandas() - main_dates.maintenance_date = main_dates.maintenance_date.map(lambda x: [x]) - main_dates.index.name = "dgst" - return main_dates.groupby("related_cert_digest").maintenance_date.agg("sum").rename("maintenance_dates") - - -class CCSchemeDataset: - @staticmethod - def _download_page(url, session=None): - if session: - conn = session - else: - conn = requests - resp = conn.get(url, headers={"User-Agent": "seccerts.org"}) - if resp.status_code != requests.codes.ok: - raise ValueError(f"Unable to download: status={resp.status_code}") - return BeautifulSoup(resp.content, "html5lib") - - @staticmethod - def get_australia_in_evaluation(): - # TODO: Information could be expanded by following url. - soup = CCSchemeDataset._download_page(constants.CC_AUSTRALIA_CERTIFIED_URL) - header = soup.find("h2", text="Products in evaluation") - table = header.find_next_sibling("table") - results = [] - for tr in table.find_all("tr"): - tds = tr.find_all("td") - if not tds: - continue - cert = { - "vendor": sns(tds[0].text), - "product": sns(tds[1].text), - "url": constants.CC_AUSTRALIA_BASE_URL + tds[1].find("a")["href"], - "level": sns(tds[2].text), - } - results.append(cert) - return results - - @staticmethod - def get_canada_certified(): - soup = CCSchemeDataset._download_page(constants.CC_CANADA_CERTIFIED_URL) - tbody = soup.find("table").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - if not tds: - continue - cert = { - "product": sns(tds[0].text), - "vendor": sns(tds[1].text), - "level": sns(tds[2].text), - "certification_date": sns(tds[3].text), - } - results.append(cert) - return results - - @staticmethod - def get_canada_in_evaluation(): - soup = CCSchemeDataset._download_page(constants.CC_CANADA_INEVAL_URL) - tbody = soup.find("table").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - if not tds: - continue - cert = { - "product": sns(tds[0].text), - "vendor": sns(tds[1].text), - "level": sns(tds[2].text), - "cert_lab": sns(tds[3].text), - } - results.append(cert) - return results - - @staticmethod - def get_france_certified(): - # TODO: Information could be expanded by following product link. - base_soup = CCSchemeDataset._download_page(constants.CC_ANSSI_CERTIFIED_URL) - category_nav = base_soup.find("ul", class_="nav-categories") - results = [] - for li in category_nav.find_all("li"): - a = li.find("a") - url = a["href"] - category_name = sns(a.text) - soup = CCSchemeDataset._download_page(constants.CC_ANSSI_BASE_URL + url) - table = soup.find("table", class_="produits-liste cc") - if not table: - continue - tbody = table.find("tbody") - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - if not tds: - continue - cert = { - "product": sns(tds[0].text), - "vendor": sns(tds[1].text), - "level": sns(tds[2].text), - "id": sns(tds[3].text), - "certification_date": sns(tds[4].text), - "category": category_name, - "url": constants.CC_ANSSI_BASE_URL + tds[0].find("a")["href"], - } - results.append(cert) - return results - - @staticmethod - def get_germany_certified(): - # TODO: Information could be expanded by following url. - base_soup = CCSchemeDataset._download_page(constants.CC_BSI_CERTIFIED_URL) - category_nav = base_soup.find("ul", class_="no-bullet row") - results = [] - for li in category_nav.find_all("li"): - a = li.find("a") - url = a["href"] - category_name = sns(a.text) - soup = CCSchemeDataset._download_page(constants.CC_BSI_BASE_URL + url) - content = soup.find("div", class_="content").find("div", class_="column") - for table in content.find_all("table"): - tbody = table.find("tbody") - header = table.find_parent("div", class_="wrapperTable").find_previous_sibling("h2") - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - if len(tds) != 4: - continue - cert = { - "cert_id": sns(tds[0].text), - "product": sns(tds[1].text), - "vendor": sns(tds[2].text), - "certification_date": sns(tds[3].text), - "category": category_name, - "url": constants.CC_BSI_BASE_URL + tds[0].find("a")["href"], - } - if header is not None: - cert["subcategory"] = sns(header.text) - results.append(cert) - return results - - @staticmethod - def get_india_certified(): - pages = {0} - seen_pages = set() - results = [] - while pages: - page = pages.pop() - seen_pages.add(page) - url = constants.CC_INDIA_CERTIFIED_URL + f"?page={page}" - soup = CCSchemeDataset._download_page(url) - - # Update pages - pager = soup.find("ul", class_="pager") - for li in pager.find_all("li"): - try: - new_page = int(li.text) - except Exception: - continue - if new_page not in seen_pages: - pages.add(new_page) - - # Parse table - tbody = soup.find("div", class_="content").find("table").find("tbody") - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - if not tds: - continue - report_a = tds[5].find("a") - target_a = tds[6].find("a") - cert_a = tds[7].find("a") - cert = { - "serial_number": sns(tds[0].text), - "product": sns(tds[1].text), - "sponsor": sns(tds[2].text), - "developer": sns(tds[3].text), - "level": sns(tds[4].text), - "report_link": report_a["href"], - "report_name": sns(report_a.text), - "target_link": target_a["href"], - "target_name": sns(target_a.text), - "cert_link": cert_a["href"], - "cert_name": sns(cert_a.text), - } - results.append(cert) - return results - - @staticmethod - def get_india_archived(): - pages = {0} - seen_pages = set() - results = [] - while pages: - page = pages.pop() - seen_pages.add(page) - url = constants.CC_INDIA_ARCHIVED_URL + f"?page={page}" - soup = CCSchemeDataset._download_page(url) - - # Update pages - pager = soup.find("ul", class_="pager") - for li in pager.find_all("li"): - try: - new_page = int(li.text) - except Exception: - continue - if new_page not in seen_pages: - pages.add(new_page) - - # Parse table - tbody = soup.find("div", class_="content").find("table").find("tbody") - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - if not tds: - continue - report_a = tds[5].find("a") - target_a = tds[6].find("a") - cert_a = tds[7].find("a") - cert = { - "serial_number": sns(tds[0].text), - "product": sns(tds[1].text), - "sponsor": sns(tds[2].text), - "developer": sns(tds[3].text), - "level": sns(tds[4].text), - "report_link": report_a["href"], - "report_name": sns(report_a.text), - "target_link": target_a["href"], - "target_name": sns(target_a.text), - "cert_link": cert_a["href"], - "cert_name": sns(cert_a.text), - "certification_date": sns(tds[8].text), - } - results.append(cert) - return results - - @staticmethod - def get_italy_certified(): # noqa: C901 - soup = CCSchemeDataset._download_page(constants.CC_ITALY_CERTIFIED_URL) - div = soup.find("div", class_="certificati") - results = [] - for cert_div in div.find_all("div", recursive=False): - title = cert_div.find("h3").text - data_div = cert_div.find("div", class_="collapse") - cert = {"title": title} - for data_p in data_div.find_all("p"): - p_text = sns(data_p.text) - if ":" not in p_text: - continue - p_name, p_data = p_text.split(":") - p_data = p_data - p_link = data_p.find("a") - if "Fornitore" in p_name: - cert["supplier"] = p_data - elif "Livello di garanzia" in p_name: - cert["level"] = p_data - elif "Data emissione certificato" in p_name: - cert["certification_date"] = p_data - elif "Data revisione" in p_name: - cert["revision_date"] = p_data - elif "Rapporto di Certificazione" in p_name and p_link: - cert["report_link_it"] = constants.CC_ITALY_BASE_URL + p_link["href"] - elif "Certification Report" in p_name and p_link: - cert["report_link_en"] = constants.CC_ITALY_BASE_URL + p_link["href"] - elif "Traguardo di Sicurezza" in p_name and p_link: - cert["target_link"] = constants.CC_ITALY_BASE_URL + p_link["href"] - elif "Nota su" in p_name and p_link: - cert["vulnerability_note_link"] = constants.CC_ITALY_BASE_URL + p_link["href"] - elif "Nota di chiarimento" in p_name and p_link: - cert["clarification_note_link"] = constants.CC_ITALY_BASE_URL + p_link["href"] - results.append(cert) - return results - - @staticmethod - def get_italy_in_evaluation(): - soup = CCSchemeDataset._download_page(constants.CC_ITALY_INEVAL_URL) - div = soup.find("div", class_="valutazioni") - results = [] - for cert_div in div.find_all("div", recursive=False): - title = cert_div.find("h3").text - data_div = cert_div.find("div", class_="collapse") - cert = {"title": title} - for data_p in data_div.find_all("p"): - p_text = sns(data_p.text) - if ":" not in p_text: - continue - p_name, p_data = p_text.split(":") - p_data = p_data - if "Committente" in p_name: - cert["client"] = p_data - elif "Livello di garanzia" in p_name: - cert["level"] = p_data - elif "Tipologia prodotto" in p_name: - cert["product_type"] = p_data - results.append(cert) - return results - - @staticmethod - def get_japan_certified(): - # TODO: Information could be expanded by following toe link. - soup = CCSchemeDataset._download_page(constants.CC_JAPAN_CERTIFIED_URL) - table = soup.find("div", id="cert_list").find("table") - results = [] - trs = list(table.find_all("tr")) - for tr in trs: - tds = tr.find_all("td") - if not tds: - continue - if len(tds) == 6: - cert = { - "cert_id": sns(tds[0].text), - "supplier": sns(tds[1].text), - "toe_overseas_name": sns(tds[2].text), - "certification_date": sns(tds[3].text), - "claim": sns(tds[4].text), - } - toe_a = tds[2].find("a") - if toe_a and "href" in toe_a.attrs: - cert["toe_overseas_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] - results.append(cert) - if len(tds) == 1: - cert = results[-1] - cert["toe_japan_name"] = sns(tds[0].text) - toe_a = tds[0].find("a") - if toe_a and "href" in toe_a.attrs: - cert["toe_japan_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] - return results - - @staticmethod - def get_japan_archived(): - # TODO: Information could be expanded by following toe link. - soup = CCSchemeDataset._download_page(constants.CC_JAPAN_ARCHIVED_URL) - table = soup.find("table") - results = [] - trs = list(table.find_all("tr")) - for tr in trs: - tds = tr.find_all("td") - if not tds: - continue - if len(tds) == 6: - cert = { - "cert_id": sns(tds[0].text), - "supplier": sns(tds[1].text), - "toe_overseas_name": sns(tds[2].text), - "certification_date": sns(tds[3].text), - "claim": sns(tds[4].text), - } - toe_a = tds[2].find("a") - if toe_a and "href" in toe_a.attrs: - cert["toe_overseas_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] - results.append(cert) - if len(tds) == 1: - cert = results[-1] - cert["toe_japan_name"] = sns(tds[0].text) - toe_a = tds[0].find("a") - if toe_a and "href" in toe_a.attrs: - cert["toe_japan_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] - return results - - @staticmethod - def get_japan_in_evaluation(): - # TODO: Information could be expanded by following toe link. - soup = CCSchemeDataset._download_page(constants.CC_JAPAN_INEVAL_URL) - table = soup.find("table") - results = [] - for tr in table.find_all("tr"): - tds = tr.find_all("td") - if not tds: - continue - toe_a = tds[1].find("a") - cert = { - "supplier": sns(tds[0].text), - "toe_name": sns(toe_a.text), - "toe_link": constants.CC_JAPAN_BASE_URL + "/" + toe_a["href"], - "claim": sns(tds[2].text), - } - results.append(cert) - return results - - @staticmethod - def get_malaysia_certified(): - soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_CERTIFIED_URL) - main_div = soup.find("div", attrs={"itemprop": "articleBody"}) - tables = main_div.find_all("table", recursive=False) - results = [] - for table in tables: - category_name = sns(table.find_previous_sibling("h3").text) - for tr in table.find_all("tr")[1:]: - tds = tr.find_all("td") - if len(tds) != 6: - continue - cert = { - "category": category_name, - "level": sns(tds[0].text), - "cert_id": sns(tds[1].text), - "certification_date": sns(tds[2].text), - "product": sns(tds[3].text), - "developer": sns(tds[4].text), - } - results.append(cert) - return results - - @staticmethod - def get_malaysia_in_evaluation(): - soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_INEVAL_URL) - main_div = soup.find("div", attrs={"itemprop": "articleBody"}) - tables = main_div.find_all("table", recursive=False) - results = [] - for table in tables: - category_name = sns(table.find_previous_sibling("h3").text) - for tr in table.find_all("tr")[1:]: - tds = tr.find_all("td") - if len(tds) != 5: - continue - cert = { - "category": category_name, - "level": sns(tds[0].text), - "project_id": sns(tds[1].text), - "toe_name": sns(tds[2].text), - "developer": sns(tds[3].text), - "expected_completion": sns(tds[4].text), - } - results.append(cert) - return results - - @staticmethod - def get_netherlands_certified(): - soup = CCSchemeDataset._download_page(constants.CC_NETHERLANDS_CERTIFIED_URL) - main_div = soup.select("body > main > div > div > div > div:nth-child(2) > div.col-lg-9 > div:nth-child(3)")[0] - rows = main_div.find_all("div", class_="row", recursive=False) - modals = main_div.find_all("div", class_="modal", recursive=False) - results = [] - for row, modal in zip(rows, modals): - row_entries = row.find_all("a") - modal_trs = modal.find_all("tr") - cert = { - "manufacturer": sns(row_entries[0].text), - "product": sns(row_entries[1].text), - "scheme": sns(row_entries[2].text), - "cert_id": sns(row_entries[3].text), - } - for tr in modal_trs: - th_text = tr.find("th").text - td = tr.find("td") - if "Manufacturer website" in th_text: - cert["manufacturer_link"] = td.find("a")["href"] - elif "Assurancelevel" in th_text: - cert["level"] = sns(td.text) - elif "Certificate" in th_text: - cert["cert_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] - elif "Certificationreport" in th_text: - cert["report_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] - elif "Securitytarget" in th_text: - cert["target_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] - elif "Maintenance report" in th_text: - cert["maintenance_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] - results.append(cert) - return results - - @staticmethod - def get_netherlands_in_evaluation(): - soup = CCSchemeDataset._download_page(constants.CC_NETHERLANDS_INEVAL_URL) - table = soup.find("table") - results = [] - for tr in table.find_all("tr")[1:]: - tds = tr.find_all("td") - cert = { - "developer": sns(tds[0].text), - "product": sns(tds[1].text), - "category": sns(tds[2].text), - "level": sns(tds[3].text), - "certification_id": sns(tds[4].text), - } - results.append(cert) - return results - - @staticmethod - def _get_norway(url): - # TODO: Information could be expanded by following product link. - soup = CCSchemeDataset._download_page(url) - results = [] - for tr in soup.find_all("tr", class_="certified-product"): - tds = tr.find_all("td") - cert = { - "product": sns(tds[0].text), - "product_link": tds[0].find("a")["href"], - "category": sns(tds[1].find("p", class_="value").text), - "developer": sns(tds[2].find("p", class_="value").text), - "certification_date": sns(tds[3].find("time").text), - } - results.append(cert) - return results - - @staticmethod - def get_norway_certified(): - return CCSchemeDataset._get_norway(constants.CC_NORWAY_CERTIFIED_URL) - - @staticmethod - def get_norway_archived(): - return CCSchemeDataset._get_norway(constants.CC_NORWAY_ARCHIVED_URL) - - @staticmethod - def _get_korea(product_class): - # TODO: Information could be expanded by following product link. - session = requests.session() - session.get(constants.CC_KOREA_EN_URL) - # Get base page - url = constants.CC_KOREA_CERTIFIED_URL + f"?product_class={product_class}" - soup = CCSchemeDataset._download_page(url, session=session) - seen_pages = set() - pages = {1} - results = [] - while pages: - page = pages.pop() - csrf = soup.find("form", id="fm").find("input", attrs={"name": "csrf"})["value"] - resp = session.post(url, data={"csrf": csrf, "selectPage": page, "product_class": product_class}) - soup = BeautifulSoup(resp.content, "html5lib") - tbody = soup.find("table", class_="cpl").find("tbody") - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - if len(tds) != 6: - continue - link = tds[0].find("a") - id = link["id"].split("-")[1] - cert = { - "product": sns(tds[0].text), - "cert_id": sns(tds[1].text), - "product_link": constants.CC_KOREA_PRODUCT_URL.format(id), - "vendor": sns(tds[2].text), - "level": sns(tds[3].text), - "category": sns(tds[4].text), - "certification_date": sns(tds[5].text), - } - results.append(cert) - seen_pages.add(page) - page_links = soup.find("div", class_="paginate").find_all("a", class_="number_off") - for page_link in page_links: - try: - new_page = int(page_link.text) - if new_page not in seen_pages: - pages.add(new_page) - except Exception: - pass - return results - - @staticmethod - def get_korea_certified(): - return CCSchemeDataset._get_korea(product_class=1) - - @staticmethod - def get_korea_suspended(): - return CCSchemeDataset._get_korea(product_class=2) - - @staticmethod - def get_korea_archived(): - return CCSchemeDataset._get_korea(product_class=4) - - @staticmethod - def _get_singapore(url): - soup = CCSchemeDataset._download_page(url) - table = soup.find("table") - skip = False - results = [] - category_name = None - for tr in table.find_all("tr"): - if skip: - skip = False - continue - tds = tr.find_all("td") - if len(tds) == 1: - category_name = sns(tds[0].text) - skip = True - continue - - cert = { - "product": sns(tds[0].text.split()[0]), - "vendor": sns(tds[1].text), - "level": sns(tds[2].text), - "certification_date": sns(tds[3].text), - "expiration_date": sns(tds[4].text), - "category": category_name, - } - for link in tds[0].find_all("a"): - link_text = sns(link.text) - if link_text == "Certificate": - cert["cert_link"] = constants.CC_SINGAPORE_BASE_URL + link["href"] - elif link_text in ("Certificate Report", "Certification Report"): - cert["report_link"] = constants.CC_SINGAPORE_BASE_URL + link["href"] - elif link_text == "Security Target": - cert["target_link"] = constants.CC_SINGAPORE_BASE_URL + link["href"] - results.append(cert) - return results - - @staticmethod - def get_singapore_certified(): - return CCSchemeDataset._get_singapore(constants.CC_SINGAPORE_CERTIFIED_URL) - - @staticmethod - def get_singapore_in_evaluation(): - soup = CCSchemeDataset._download_page(constants.CC_SINGAPORE_CERTIFIED_URL) - header = soup.find(lambda x: x.name == "h3" and x.text == "In Evaluation") - table = header.find_next("table") - results = [] - for tr in table.find_all("tr")[1:]: - tds = tr.find_all("td") - cert = { - "name": sns(tds[0].text), - "vendor": sns(tds[1].text), - "level": sns(tds[2].text), - } - results.append(cert) - return results - - @staticmethod - def get_singapore_archived(): - return CCSchemeDataset._get_singapore(constants.CC_SINGAPORE_ARCHIVED_URL) - - @staticmethod - def get_spain_certified(): - soup = CCSchemeDataset._download_page(constants.CC_SPAIN_CERTIFIED_URL) - tbody = soup.find("table", class_="djc_items_table").find("tbody") - results = [] - for tr in tbody.find_all("tr", recursive=False): - tds = tr.find_all("td") - cert = { - "product": sns(tds[0].text), - "product_link": constants.CC_SPAIN_BASE_URL + tds[0].find("a")["href"], - "category": sns(tds[1].text), - "manufacturer": sns(tds[2].text), - "certification_date": sns(tds[3].find("td", class_="djc_value").text), - } - results.append(cert) - return results - - @staticmethod - def _get_sweden(url): - # TODO: Information could be expanded by following product link. - soup = CCSchemeDataset._download_page(url) - nav = soup.find("main").find("nav", class_="component-nav-box__list") - results = [] - for link in nav.find_all("a"): - cert = {"product": sns(link.text), "product_link": constants.CC_SWEDEN_BASE_URL + link["href"]} - results.append(cert) - return results - - @staticmethod - def get_sweden_certified(): - return CCSchemeDataset._get_sweden(constants.CC_SWEDEN_CERTIFIED_URL) - - @staticmethod - def get_sweden_in_evaluation(): - return CCSchemeDataset._get_sweden(constants.CC_SWEDEN_INEVAL_URL) - - @staticmethod - def get_sweden_archived(): - return CCSchemeDataset._get_sweden(constants.CC_SWEDEN_ARCHIVED_URL) - - @staticmethod - def get_turkey_certified(): - results = [] - with tempfile.TemporaryDirectory() as tmpdir: - pdf_path = Path(tmpdir) / "turkey.pdf" - resp = requests.get(constants.CC_TURKEY_ARCHIVED_URL) - if resp.status_code != requests.codes.ok: - raise ValueError(f"Unable to download: status={resp.status_code}") - with pdf_path.open("wb") as f: - f.write(resp.content) - dfs = tabula.read_pdf(str(pdf_path), pages="all") - for df in dfs: - for line in df.values: - cert = { - # TODO: Split item number and generate several dicts for a range they include. - "item_no": line[0], - "developer": line[1], - "product": line[2], - "cc_version": line[3], - "level": line[4], - "cert_lab": line[5], - "certification_date": line[6], - "expiration_date": line[7], - # TODO: Parse "Ongoing Evaluation" out of this field as well. - "archived": isinstance(line[9], str) and "Archived" in line[9], - } - results.append(cert) - return results - - @staticmethod - def get_usa_certified(): - # TODO: Information could be expanded by following product link. - # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). - soup = CCSchemeDataset._download_page(constants.CC_USA_CERTIFIED_URL) - tbody = soup.find("table", class_="tablesorter").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - vendor_span = tds[0].find("span", class_="b u") - product_link = tds[0].find("a") - scheme_img = tds[6].find("img") - # Only return the US certifications. - if scheme_img["title"] != "USA": - continue - cert = { - "product": sns(product_link.text), - "vendor": sns(vendor_span.text), - "product_link": product_link["href"], - "id": sns(tds[1].text), - "cc_claim": sns(tds[2].text), - "cert_lab": sns(tds[3].text), - "certification_date": sns(tds[4].text), - "assurance_maintenance_date": sns(tds[5].text), - } - results.append(cert) - return results - - @staticmethod - def get_usa_in_evaluation(): - # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). - soup = CCSchemeDataset._download_page(constants.CC_USA_INEVAL_URL) - tbody = soup.find("table", class_="tablesorter").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - vendor_span = tds[0].find("span", class_="b u") - product_name = None - for child in tds[0].children: - if isinstance(child, NavigableString): - product_name = sns(child) - break - cert = { - "vendor": sns(vendor_span.text), - "id": sns(tds[1].text), - "cc_claim": sns(tds[2].text), - "cert_lab": sns(tds[3].text), - "kickoff_date": sns(tds[4].text), - } - if product_name: - cert["product"] = product_name - results.append(cert) - return results - - @staticmethod - def get_usa_archived(): - # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). - soup = CCSchemeDataset._download_page(constants.CC_USA_ARCHIVED_URL) - tbody = soup.find("table", class_="tablesorter").find("tbody") - results = [] - for tr in tbody.find_all("tr"): - tds = tr.find_all("td") - scheme_img = tds[5].find("img") - # Only return the US certifications. - if scheme_img["title"] != "USA": - continue - vendor_span = tds[0].find("span", class_="b u") - product_name = None - for child in tds[0].children: - if isinstance(child, NavigableString): - product_name = sns(child) - break - cert = { - "vendor": sns(vendor_span.text), - "id": sns(tds[1].text), - "cc_claim": sns(tds[2].text), - "cert_lab": sns(tds[3].text), - "certification_date": sns(tds[4].text), - } - if product_name: - cert["product"] = product_name - results.append(cert) - return results diff --git a/sec_certs/dataset/cpe.py b/sec_certs/dataset/cpe.py deleted file mode 100644 index 927ce674..00000000 --- a/sec_certs/dataset/cpe.py +++ /dev/null @@ -1,197 +0,0 @@ -from __future__ import annotations - -import copy -import itertools -import logging -import tempfile -import xml.etree.ElementTree as ET -import zipfile -from pathlib import Path -from typing import ClassVar, Iterator - -import pandas as pd - -from sec_certs import constants -from sec_certs.dataset.cve import CVEDataset -from sec_certs.dataset.json_path_dataset import JSONPathDataset -from sec_certs.sample.cpe import CPE, cached_cpe -from sec_certs.serialization.json import ComplexSerializableType, serialize -from sec_certs.utils import helpers -from sec_certs.utils.tqdm import tqdm - -logger = logging.getLogger(__name__) - - -class CPEDataset(JSONPathDataset, ComplexSerializableType): - """ - Dataset of CPE records. Includes look-up dictionaries for fast search. - """ - - CPE_XML_BASENAME: ClassVar[str] = "official-cpe-dictionary_v2.3.xml" - CPE_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/xml/cpe/dictionary/" + CPE_XML_BASENAME + ".zip" - - def __init__( - self, - was_enhanced_with_vuln_cpes: bool, - cpes: dict[str, CPE], - json_path: str | Path = constants.DUMMY_NONEXISTING_PATH, - ): - self.was_enhanced_with_vuln_cpes = was_enhanced_with_vuln_cpes - self.cpes = cpes - self.json_path = Path(json_path) - - self.vendor_to_versions: dict[str, set[str]] = dict() - self.vendor_version_to_cpe: dict[tuple[str, str], set[CPE]] = dict() - self.title_to_cpes: dict[str, set[CPE]] = dict() - self.vendors: set[str] = set() - - self.build_lookup_dicts() - - def __iter__(self) -> Iterator[CPE]: - yield from self.cpes.values() - - def __getitem__(self, item: str) -> CPE: - return self.cpes.__getitem__(item.lower()) - - def __setitem__(self, key: str, value: CPE) -> None: - self.cpes.__setitem__(key.lower(), value) - - def __len__(self) -> int: - return len(self.cpes) - - def __contains__(self, item: CPE) -> bool: - if not isinstance(item, CPE): - raise ValueError(f"{item} is not of CPE class") - return item.uri in self.cpes.keys() and self.cpes[item.uri] == item - - def __eq__(self, other: object) -> bool: - return isinstance(other, CPEDataset) and self.cpes == other.cpes - - @property - def serialized_attributes(self) -> list[str]: - return ["was_enhanced_with_vuln_cpes", "cpes"] - - def build_lookup_dicts(self) -> None: - """ - Will build look-up dictionaries that are used for fast matching. - """ - logger.info("CPE dataset: building lookup dictionaries.") - self.vendor_to_versions = {x.vendor: set() for x in self} - self.vendor_version_to_cpe = dict() - self.title_to_cpes = dict() - self.vendors = set(self.vendor_to_versions.keys()) - for cpe in self: - self.vendor_to_versions[cpe.vendor].add(cpe.version) - if (cpe.vendor, cpe.version) not in self.vendor_version_to_cpe: - self.vendor_version_to_cpe[(cpe.vendor, cpe.version)] = {cpe} - else: - self.vendor_version_to_cpe[(cpe.vendor, cpe.version)].add(cpe) - - if cpe.title: - if cpe.title not in self.title_to_cpes: - self.title_to_cpes[cpe.title] = {cpe} - else: - self.title_to_cpes[cpe.title].add(cpe) - - @classmethod - def from_web(cls, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> CPEDataset: - """ - Creates CPEDataset from NIST resources published on-line - - :param Union[str, Path] json_path: Path to store the dataset to - :return CPEDataset: The resulting dataset - """ - with tempfile.TemporaryDirectory() as tmp_dir: - xml_path = Path(tmp_dir) / cls.CPE_XML_BASENAME - zip_path = Path(tmp_dir) / (cls.CPE_XML_BASENAME + ".zip") - helpers.download_file(cls.CPE_URL, zip_path) - - with zipfile.ZipFile(zip_path, "r") as zip_ref: - zip_ref.extractall(tmp_dir) - - return cls._from_xml(xml_path, json_path) - - @classmethod - def _from_xml(cls, xml_path: str | Path, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> CPEDataset: - logger.info("Loading CPE dataset from XML.") - root = ET.parse(xml_path).getroot() - dct = {} - for cpe_item in root.findall("{http://cpe.mitre.org/dictionary/2.0}cpe-item"): - found_title = cpe_item.find("{http://cpe.mitre.org/dictionary/2.0}title") - if found_title is None: - raise RuntimeError( - "Title is not found during building CPE dataset from xml - this should not be happening" - ) - title = found_title.text - - found_cpe_uri = cpe_item.find("{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item") - if found_cpe_uri is None: - raise RuntimeError( - "CPE uri is not found during building CPE dataset from xml - this should not be happening" - ) - cpe_uri = found_cpe_uri.attrib["name"] - - dct[cpe_uri] = cached_cpe(cpe_uri, title) - - return cls(False, dct, json_path) - - def to_pandas(self) -> pd.DataFrame: - """ - Turns the dataset into pandas DataFrame. Each CPE record forms a row. - - :return pd.DataFrame: the resulting DataFrame - """ - df = pd.DataFrame([x.pandas_tuple for x in self], columns=CPE.pandas_columns) - df = df.set_index("uri") - return df - - @serialize - def enhance_with_cpes_from_cve_dataset(self, cve_dset: CVEDataset | str | Path) -> None: - """ - Some CPEs are present only in the CVEDataset and are missing from the CPE Dataset. - This method goes through the provided CVEDataset and enriches self with CPEs from - the CVEDataset. - - :param Union[CVEDataset, str, Path] cve_dset: CVEDataset of a path to it. - """ - - def _adding_condition( - considered_cpe: CPE, - vndr_item_lookup: set[tuple[str, str]], - vndr_item_version_lookup: set[tuple[str, str, str]], - ) -> bool: - if ( - considered_cpe.version == constants.CPE_VERSION_NA - and (considered_cpe.vendor, considered_cpe.item_name) not in vndr_item_lookup - ): - return True - elif ( - considered_cpe.version != constants.CPE_VERSION_NA - and (considered_cpe.vendor, considered_cpe.item_name, considered_cpe.version) - not in vndr_item_version_lookup - ): - return True - return False - - if isinstance(cve_dset, (str, Path)): - cve_dset = CVEDataset.from_json(cve_dset) - - if not isinstance(cve_dset, CVEDataset): - raise RuntimeError("Conversion of CVE dataset did not work.") - all_cpes_in_cve_dset = set(itertools.chain.from_iterable(cve.vulnerable_cpes for cve in cve_dset)) - - old_len = len(self.cpes) - - # We only enrich if tuple (vendor, item_name) is not already in the dataset - vendor_item_lookup = {(cpe.vendor, cpe.item_name) for cpe in self} - vendor_item_version_lookup = {(cpe.vendor, cpe.item_name, cpe.version) for cpe in self} - for cpe in tqdm(all_cpes_in_cve_dset, desc="Enriching CPE dataset with new CPEs"): - if _adding_condition(cpe, vendor_item_lookup, vendor_item_version_lookup): - new_cpe = copy.deepcopy(cpe) - new_cpe.start_version = None - new_cpe.end_version = None - self[new_cpe.uri] = new_cpe - self.build_lookup_dicts() - - logger.info(f"Enriched the CPE dataset with {len(self.cpes) - old_len} new CPE records.") - self.was_enhanced_with_vuln_cpes = True diff --git a/sec_certs/dataset/cve.py b/sec_certs/dataset/cve.py deleted file mode 100644 index b65c81c6..00000000 --- a/sec_certs/dataset/cve.py +++ /dev/null @@ -1,221 +0,0 @@ -from __future__ import annotations - -import datetime -import glob -import itertools -import json -import logging -import shutil -import tempfile -import zipfile -from pathlib import Path -from typing import ClassVar - -import numpy as np -import pandas as pd - -import sec_certs.constants as constants -import sec_certs.utils.helpers as helpers -from sec_certs.config.configuration import config -from sec_certs.dataset.json_path_dataset import JSONPathDataset -from sec_certs.sample.cpe import CPE, cached_cpe -from sec_certs.sample.cve import CVE -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.utils.parallel_processing import process_parallel -from sec_certs.utils.tqdm import tqdm - -logger = logging.getLogger(__name__) - - -class CVEDataset(JSONPathDataset, ComplexSerializableType): - CVE_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" - CPE_MATCH_FEED_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip" - - def __init__(self, cves: dict[str, CVE], json_path: str | Path = constants.DUMMY_NONEXISTING_PATH): - self.cves = cves - self.json_path = Path(json_path) - self.cpe_to_cve_ids_lookup: dict[str, set[str]] = dict() - - @property - def serialized_attributes(self) -> list[str]: - return ["cves"] - - def __iter__(self): - yield from self.cves.values() - - def __getitem__(self, item: str) -> CVE: - return self.cves.__getitem__(item.upper()) - - def __setitem__(self, key: str, value: CVE): - self.cves.__setitem__(key.lower(), value) - - def __len__(self) -> int: - return len(self.cves) - - def __eq__(self, other: object): - return isinstance(other, CVEDataset) and self.cves == other.cves - - def build_lookup_dict(self, use_nist_mapping: bool = True, nist_matching_filepath: Path | None = None): - """ - Builds look-up dictionary CPE -> Set[CVE] - Developer's note: There are 3 CPEs that are present in the cpe matching feed, but are badly processed by CVE - feed, in which case they won't be found as a key in the dictionary. We intentionally ignore those. Feel free - to add corner cases and manual fixes. According to our investigation, the suffereing CPEs are: - - CPE(uri='cpe:2.3:a:arubanetworks:airwave:*:*:*:*:*:*:*:*', title=None, version='*', vendor='arubanetworks', item_name='airwave', start_version=None, end_version=('excluding', '8.2.0.0')) - - CPE(uri='cpe:2.3:a:bayashi:dopvcomet\\*:0009:b:*:*:*:*:*:*', title=None, version='0009', vendor='bayashi', item_name='dopvcomet\\*', start_version=None, end_version=None) - - CPE(uri='cpe:2.3:a:bayashi:dopvstar\\*:0091:*:*:*:*:*:*:*', title=None, version='0091', vendor='bayashi', item_name='dopvstar\\*', start_version=None, end_version=None) - """ - self.cpe_to_cve_ids_lookup = dict() - self.cves = {x.cve_id.upper(): x for x in self} - - logger.info("Getting CPE matching dictionary from NIST.gov") - - if use_nist_mapping: - matching_dict = self.get_nist_cpe_matching_dict(nist_matching_filepath) - - cve: CVE - for cve in tqdm(self, desc="Building-up lookup dictionaries for fast CVE matching"): - # See note above, we use matching_dict.get(cpe, []) instead of matching_dict[cpe] as would be expected - if use_nist_mapping: - vulnerable_configurations = list( - itertools.chain.from_iterable(matching_dict.get(cpe, []) for cpe in cve.vulnerable_cpes) - ) - else: - vulnerable_configurations = cve.vulnerable_cpes - for cpe in vulnerable_configurations: - if cpe.uri not in self.cpe_to_cve_ids_lookup: - self.cpe_to_cve_ids_lookup[cpe.uri] = {cve.cve_id} - else: - self.cpe_to_cve_ids_lookup[cpe.uri].add(cve.cve_id) - - @classmethod - def download_cves(cls, output_path_str: str, start_year: int, end_year: int): - output_path = Path(output_path_str) - if not output_path.exists: - output_path.mkdir() - - urls = [cls.CVE_URL + str(x) + ".json.zip" for x in range(start_year, end_year + 1)] - - logger.info(f"Identified {len(urls)} CVE files to fetch from nist.gov. Downloading them into {output_path}") - with tempfile.TemporaryDirectory() as tmp_dir: - outpaths = [Path(tmp_dir) / Path(x).name.rstrip(".zip") for x in urls] - responses = helpers.download_parallel(urls, outpaths, "Downloading CVEs resources from NVD") - - for o, r in zip(outpaths, responses): - if r == constants.RESPONSE_OK: - with zipfile.ZipFile(o, "r") as zip_handle: - zip_handle.extractall(output_path) - - @classmethod - def from_nist_json(cls, input_path: str) -> CVEDataset: - with Path(input_path).open("r") as handle: - data = json.load(handle) - cves = [CVE.from_nist_dict(x) for x in data["CVE_Items"]] - return cls({x.cve_id: x for x in cves}) - - @classmethod - def from_web( - cls, - start_year: int = 2002, - end_year: int = datetime.datetime.now().year, - json_path: str | Path = constants.DUMMY_NONEXISTING_PATH, - ): - logger.info("Building CVE dataset from nist.gov website.") - with tempfile.TemporaryDirectory() as tmp_dir: - cls.download_cves(tmp_dir, start_year, end_year) - json_files = glob.glob(tmp_dir + "/*.json") - - all_cves = dict() - logger.info("Downloaded required resources. Building CVEDataset from jsons.") - results = process_parallel( - cls.from_nist_json, - json_files, - config.n_threads, - use_threading=False, - progress_bar_desc="Building CVEDataset from jsons", - ) - for r in results: - all_cves.update(r.cves) - - return cls(all_cves, json_path) - - def get_cve_ids_for_cpe_uri(self, cpe_uri: str) -> set[str] | None: - return self.cpe_to_cve_ids_lookup.get(cpe_uri, None) - - def filter_related_cpes(self, relevant_cpes: set[CPE]): - """ - Since each of the CVEs is related to many CPEs, the dataset size explodes (serialized). For certificates, - only CPEs within sample dataset are relevant. This function modifies all CVE elements. Specifically, it - deletes all CPE records unless they are part of relevant_cpe_uris. - :param relevant_cpes: List of relevant CPEs to keep in CVE dataset. - """ - total_deleted_cpes = 0 - cve_ids_to_delete = [] - for cve in self: - n_cpes_orig = len(cve.vulnerable_cpes) - cve.vulnerable_cpes = list(filter(lambda x: x in relevant_cpes, cve.vulnerable_cpes)) - total_deleted_cpes += n_cpes_orig - len(cve.vulnerable_cpes) - if not cve.vulnerable_cpes: - cve_ids_to_delete.append(cve.cve_id) - - for cve_id in cve_ids_to_delete: - del self.cves[cve_id] - logger.info( - f"Totally deleted {total_deleted_cpes} irrelevant CPEs and {len(cve_ids_to_delete)} CVEs from CVEDataset." - ) - - def to_pandas(self) -> pd.DataFrame: - df = pd.DataFrame([x.pandas_tuple for x in self], columns=CVE.pandas_columns) - df.cwe_ids = df.cwe_ids.map(lambda x: x if x else np.nan) - return df.set_index("cve_id") - - def get_nist_cpe_matching_dict(self, input_filepath: Path | None) -> dict[CPE, list[CPE]]: - """ - Computes dictionary that maps complex CPEs to list of simple CPEs. - """ - - def parse_key_cpe(field: dict) -> CPE: - start_version = None - if "versionStartIncluding" in field: - start_version = ("including", field["versionStartIncluding"]) - elif "versionStartExcluding" in field: - start_version = ("excluding", field["versionStartExcluding"]) - - end_version = None - if "versionEndIncluding" in field: - end_version = ("including", field["versionEndIncluding"]) - elif "versionEndExcluding" in field: - end_version = ("excluding", field["versionEndExcluding"]) - - return cached_cpe(field["cpe23Uri"], start_version=start_version, end_version=end_version) - - def parse_values_cpe(field: dict) -> list[CPE]: - return [cached_cpe(x["cpe23Uri"]) for x in field["cpe_name"]] - - logger.debug("Attempting to get NIST mapping file.") - if not input_filepath or not input_filepath.is_file(): - logger.debug("NIST mapping file not available, going to download.") - with tempfile.TemporaryDirectory() as tmp_dir: - filename = Path(self.CPE_MATCH_FEED_URL).name - download_path = Path(tmp_dir) / filename - unzipped_path = Path(tmp_dir) / filename.rstrip(".zip") - helpers.download_file(self.CPE_MATCH_FEED_URL, download_path) - - with zipfile.ZipFile(download_path, "r") as zip_handle: - zip_handle.extractall(tmp_dir) - with unzipped_path.open("r") as handle: - match_data = json.load(handle) - if input_filepath: - logger.debug(f"Copying attained NIST mapping file to {input_filepath}") - shutil.move(str(unzipped_path), str(input_filepath)) - else: - with input_filepath.open("r") as handle: - match_data = json.load(handle) - - mapping_dict = dict() - for match in tqdm(match_data["matches"], desc="parsing cpe matching (by NIST) dictionary"): - key = parse_key_cpe(match) - value = parse_values_cpe(match) - mapping_dict[key] = value if value else [key] - - return mapping_dict diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py deleted file mode 100644 index 49a6b340..00000000 --- a/sec_certs/dataset/dataset.py +++ /dev/null @@ -1,572 +0,0 @@ -from __future__ import annotations - -import itertools -import json -import logging -import re -import shutil -import tempfile -from abc import ABC, abstractmethod -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -from typing import Any, Generic, Iterator, TypeVar, cast - -import pandas as pd - -import sec_certs.constants as constants -import sec_certs.utils.helpers as helpers -from sec_certs.config.configuration import config -from sec_certs.dataset.cpe import CPEDataset -from sec_certs.dataset.cve import CVEDataset -from sec_certs.model.cpe_matching import CPEClassifier -from sec_certs.sample.certificate import Certificate -from sec_certs.sample.cpe import CPE -from sec_certs.serialization.json import ComplexSerializableType, get_class_fullname, serialize -from sec_certs.utils.tqdm import tqdm - -logger = logging.getLogger(__name__) - - -@dataclass -class AuxillaryDatasets: - cpe_dset: CPEDataset | None = None - cve_dset: CVEDataset | None = None - - -CertSubType = TypeVar("CertSubType", bound=Certificate) -AuxillaryDatasetsSubType = TypeVar("AuxillaryDatasetsSubType", bound=AuxillaryDatasets) -DatasetSubType = TypeVar("DatasetSubType", bound="Dataset") - - -class Dataset(Generic[CertSubType, AuxillaryDatasetsSubType], ComplexSerializableType, ABC): - """ - Base class for dataset of certificates from CC and FIPS 140 schemes. Layouts public - functions, the processing pipeline and common operations on the dataset and certs. - """ - - @dataclass - class DatasetInternalState(ComplexSerializableType): - meta_sources_parsed: bool = False - artifacts_downloaded: bool = False - pdfs_converted: bool = False - auxillary_datasets_processed: bool = False - certs_analyzed: bool = False - - def __bool__(self): - return any(vars(self)) - - def __init__( - self, - certs: dict[str, CertSubType] = dict(), - root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH, - name: str | None = None, - description: str = None, - state: DatasetInternalState | None = None, - auxillary_datasets: AuxillaryDatasetsSubType | None = None, - ): - self.certs = certs - - self.timestamp = datetime.now() - self.sha256_digest = "not implemented" - self.name = name if name else type(self).__name__.lower() + "_dataset" - self.description = description if description else "No description provided" - self.state = state if state else self.DatasetInternalState() - - if not auxillary_datasets: - self.auxillary_datasets = AuxillaryDatasets() - else: - self.auxillary_datasets = auxillary_datasets - - self.root_dir = Path(root_dir) - - @property - def root_dir(self) -> Path: - """ - Directory that will hold the serialized dataset files. - """ - return self._root_dir - - @root_dir.setter - def root_dir(self: DatasetSubType, new_dir: str | Path) -> None: - """ - This setter will only set the root dir and all internal paths so that they point - to the new root dir. No data is being moved around. - """ - new_dir = Path(new_dir) - if new_dir.is_file(): - raise ValueError(f"Root dir of {get_class_fullname(self)} cannot be a file.") - - self._root_dir = new_dir - self._set_local_paths() - - @property - def web_dir(self) -> Path: - """ - Path to certification-artifacts posted on web. - """ - return self.root_dir / "web" - - @property - def auxillary_datasets_dir(self) -> Path: - """ - Path to directory with auxillary datasets. - """ - return self.root_dir / "auxillary_datasets" - - @property - def certs_dir(self) -> Path: - """ - Returns directory that holds files associated with certificates - """ - return self.root_dir / "certs" - - @property - def cpe_dataset_path(self) -> Path: - return self.auxillary_datasets_dir / "cpe_dataset.json" - - @property - def cve_dataset_path(self) -> Path: - return self.auxillary_datasets_dir / "cve_dataset.json" - - @property - def nist_cve_cpe_matching_dset_path(self) -> Path: - return self.auxillary_datasets_dir / "nvdcpematch-1.0.json" - - @property - def json_path(self) -> Path: - return self.root_dir / (self.name + ".json") - - def __contains__(self, item: object) -> bool: - if not isinstance(item, Certificate): - raise TypeError( - f"You attempted to check if {type(item)} is member of {type(self)}, but only {type(Certificate)} are allowed to be members." - ) - return item.dgst in self.certs - - def __iter__(self) -> Iterator[CertSubType]: - yield from self.certs.values() - - def __getitem__(self, item: str) -> CertSubType: - return self.certs.__getitem__(item.lower()) - - def __setitem__(self, key: str, value: CertSubType): - self.certs.__setitem__(key.lower(), value) - - def __len__(self) -> int: - return len(self.certs) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Dataset): - return NotImplemented - return self.certs == other.certs - - def __str__(self) -> str: - return str(type(self).__name__) + ":" + self.name + ", " + str(len(self)) + " certificates" - - @classmethod - def from_web(cls: type[DatasetSubType], url: str, progress_bar_desc: str, filename: str) -> DatasetSubType: - """ - Fetches a fully processed dataset instance from static site that hosts it. - """ - with tempfile.TemporaryDirectory() as tmp_dir: - dset_path = Path(tmp_dir) / filename - helpers.download_file(url, dset_path, show_progress_bar=True, progress_bar_desc=progress_bar_desc) - dset = cls.from_json(dset_path) - dset.root_dir = constants.DUMMY_NONEXISTING_PATH - return dset - - def to_dict(self) -> dict[str, Any]: - return { - "state": self.state, - "timestamp": self.timestamp, - "sha256_digest": self.sha256_digest, - "name": self.name, - "description": self.description, - "n_certs": len(self), - "certs": list(self.certs.values()), - } - - @classmethod - def from_dict(cls: type[DatasetSubType], dct: dict) -> DatasetSubType: - certs = {x.dgst: x for x in dct["certs"]} - dset = cls(certs, name=dct["name"], description=dct["description"], state=dct["state"]) - if len(dset) != (claimed := dct["n_certs"]): - logger.error( - f"The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed})." - ) - return dset - - @classmethod - def from_json(cls: type[DatasetSubType], input_path: str | Path) -> DatasetSubType: - dset = cast("DatasetSubType", ComplexSerializableType.from_json(input_path)) - dset._root_dir = Path(input_path).parent.absolute() - dset._set_local_paths() - return dset - - def _set_local_paths(self) -> None: - if self.auxillary_datasets.cpe_dset: - self.auxillary_datasets.cpe_dset.json_path = self.cpe_dataset_path - if self.auxillary_datasets.cve_dset: - self.auxillary_datasets.cve_dset.json_path = self.cve_dataset_path - - def move_dataset(self, new_root_dir: str | Path) -> None: - """ - Moves all dataset files to `new_root_dir` and adjusts all paths internally. Deletes the artifacts from the original location. - :param str | Path new_root_dir: path to directory where the new dataset shall be stored. - """ - new_root_dir = Path(new_root_dir) - if new_root_dir.is_file(): - raise ValueError("New root dir must be a directory, not an existing file.") - new_root_dir.mkdir(parents=True, exist_ok=True) - - shutil.copytree(str(self.root_dir), str(new_root_dir), dirs_exist_ok=True) - shutil.rmtree(self.root_dir) - self.root_dir = new_root_dir - - def copy_dataset(self, new_root_dir: str | Path) -> None: - """ - Copies all dataset files to `new_root_dir` and adjusts all paths internally. Keeps the artifacts from the original location. - :param str | Path new_root_dir: path to directory where the new dataset shall be stored. - """ - new_root_dir = Path(new_root_dir) - if new_root_dir.is_file(): - raise ValueError("New root dir must be a directory, not an existing file.") - new_root_dir.mkdir(parents=True, exist_ok=True) - - shutil.copytree(str(self.root_dir), str(new_root_dir), dirs_exist_ok=True) - self.root_dir = new_root_dir - - def _get_certs_by_name(self, name: str) -> set[CertSubType]: - """ - Returns list of certificates that match given name. - """ - return {crt for crt in self if crt.name and crt.name == name} - - @abstractmethod - def get_certs_from_web(self) -> None: - raise NotImplementedError("Not meant to be implemented by the base class.") - - @serialize - @abstractmethod - def process_auxillary_datasets(self, download_fresh: bool = False) -> None: - """ - Processes all auxillary datasets (CPE, CVE, ...) that are required during computation. - """ - logger.info("Processing auxillary datasets.") - self.auxillary_datasets_dir.mkdir(parents=True, exist_ok=True) - self.auxillary_datasets.cpe_dset = self._prepare_cpe_dataset(download_fresh) - self.auxillary_datasets.cve_dset = self._prepare_cve_dataset(download_fresh_cves=download_fresh) - self.state.auxillary_datasets_processed = True - - @serialize - def download_all_artifacts(self, fresh: bool = True) -> None: - """ - Downloads all artifacts related to certification in the given scheme. - """ - if not self.state.meta_sources_parsed: - logger.error("Attempting to download pdfs while not having csv/html meta-sources parsed. Returning.") - return - - logger.info("Attempting to download certification artifacts.") - self._download_all_artifacts_body(fresh) - if fresh: - self._download_all_artifacts_body(False) - - self.state.artifacts_downloaded = True - - @abstractmethod - def _download_all_artifacts_body(self, fresh: bool = True) -> None: - raise NotImplementedError("Not meant to be implemented by the base class.") - - @serialize - def convert_all_pdfs(self, fresh: bool = True) -> None: - """ - Converts all pdf artifacts to txt, given the certification scheme. - """ - if not self.state.artifacts_downloaded: - logger.error("Attempting to convert pdfs while not having the artifacts downloaded. Returning.") - return - - logger.info("Converting all PDFs to txt") - self._convert_all_pdfs_body(fresh) - if fresh: - self._convert_all_pdfs_body(False) - - self.state.pdfs_converted = True - - @abstractmethod - def _convert_all_pdfs_body(self, fresh: bool = True) -> None: - raise NotImplementedError("Not meant to be implemented by the base class.") - - @serialize - def analyze_certificates(self) -> None: - """ - Does two things: - - Extracts data from certificates (keywords, etc.) - - Computes various heuristics on the certificates. - """ - if not self.state.pdfs_converted: - logger.info( - "Attempting run analysis of txt files while not having the pdf->txt conversion done. Returning." - ) - return - if not self.state.auxillary_datasets_processed: - logger.info( - "Attempting to run analysis of certifies while not having the auxillary datasets processed. Returning." - ) - - logger.info("Analyzing certificates.") - self._analyze_certificates_body() - self.state.certs_analyzed = True - - def _analyze_certificates_body(self) -> None: - self.extract_data() - self._compute_heuristics() - - @abstractmethod - def extract_data(self) -> None: - raise NotImplementedError("Not meant to be implemented by the base class.") - - def _compute_heuristics(self) -> None: - logger.info("Computing various heuristics from the certificates.") - self.compute_cpe_heuristics() - self.compute_related_cves() - self._compute_references() - self._compute_transitive_vulnerabilities() - - @abstractmethod - def _compute_references(self) -> None: - raise NotImplementedError("Not meant to be implemented by the base class.") - - @abstractmethod - def _compute_transitive_vulnerabilities(self) -> None: - raise NotImplementedError("Not meant to be implemented by the base class.") - - def _prepare_cpe_dataset(self, download_fresh_cpes: bool = False) -> CPEDataset: - logger.info("Preparing CPE dataset.") - if not self.auxillary_datasets_dir.exists(): - self.auxillary_datasets_dir.mkdir(parents=True) - - if not self.cpe_dataset_path.exists() or download_fresh_cpes is True: - cpe_dataset = CPEDataset.from_web(self.cpe_dataset_path) - cpe_dataset.to_json() - else: - cpe_dataset = CPEDataset.from_json(str(self.cpe_dataset_path)) - - return cpe_dataset - - def _prepare_cve_dataset( - self, download_fresh_cves: bool = False, use_nist_cpe_matching_dict: bool = True - ) -> CVEDataset: - logger.info("Preparing CVE dataset.") - if not self.auxillary_datasets_dir.exists(): - self.auxillary_datasets_dir.mkdir(parents=True) - - if not self.cve_dataset_path.exists() or download_fresh_cves is True: - cve_dataset = CVEDataset.from_web(json_path=self.cve_dataset_path) - cve_dataset.to_json() - else: - cve_dataset = CVEDataset.from_json(self.cve_dataset_path) - - cve_dataset.build_lookup_dict(use_nist_cpe_matching_dict, self.nist_cve_cpe_matching_dset_path) - return cve_dataset - - @serialize - def compute_cpe_heuristics(self, download_fresh_cpes: bool = False) -> CPEClassifier: - """ - Computes matching CPEs for the certificates. - """ - WINDOWS_WEAK_CPES: set[CPE] = { - CPE("cpe:2.3:o:microsoft:windows:-:*:*:*:*:*:x64:*", "Microsoft Windows on X64", None, None), - CPE("cpe:2.3:o:microsoft:windows:-:*:*:*:*:*:x86:*", "Microsoft Windows on X86", None, None), - } - - def filter_condition(cpe: CPE) -> bool: - """ - Filters out very weak CPE matches that don't improve our database. - """ - if ( - cpe.title - and (cpe.version == "-" or cpe.version == "*") - and not any(char.isdigit() for char in cpe.title) - ): - return False - elif ( - not cpe.title - and cpe.item_name - and (cpe.version == "-" or cpe.version == "*") - and not any(char.isdigit() for char in cpe.item_name) - ): - return False - elif re.match(constants.RELEASE_CANDIDATE_REGEX, cpe.update): - return False - elif cpe in WINDOWS_WEAK_CPES: - return False - return True - - logger.info("Computing heuristics: Finding CPE matches for certificates") - if not self.auxillary_datasets.cpe_dset or download_fresh_cpes: - self.auxillary_datasets.cpe_dset = self._prepare_cpe_dataset(download_fresh_cpes) - - # Temporarily disabled, see: https://github.com/crocs-muni/sec-certs/issues/173 - # if not cpe_dset.was_enhanced_with_vuln_cpes: - # self.auxillary_datasets.cve_dset = self._prepare_cve_dataset(download_fresh_cves=False) - # self.auxillary_datasets.cpe_dset.enhance_with_cpes_from_cve_dataset(cve_dset) # this also calls build_lookup_dicts() on cpe_dset - # else: - # self.auxillary_datasets.cpe_dset.build_lookup_dicts() - - clf = CPEClassifier(config.cpe_matching_threshold, config.cpe_n_max_matches) - clf.fit([x for x in self.auxillary_datasets.cpe_dset if filter_condition(x)]) - - cert: CertSubType - for cert in tqdm(self, desc="Predicting CPE matches with the classifier"): - cert.compute_heuristics_version() - - cert.heuristics.cpe_matches = ( - clf.predict_single_cert(cert.manufacturer, cert.name, cert.heuristics.extracted_versions) - if cert.name - else None - ) - - return clf - - def to_label_studio_json(self, output_path: str | Path) -> None: - cpe_dset = self._prepare_cpe_dataset() - - lst = [] - for cert in [x for x in cast(Iterator[Certificate], self) if x.heuristics.cpe_matches]: - dct = {"text": cert.label_studio_title} - candidates = [cpe_dset[x].title for x in cert.heuristics.cpe_matches] - candidates += ["No good match"] * (config.cpe_n_max_matches - len(candidates)) - options = ["option_" + str(x) for x in range(1, config.cpe_n_max_matches)] - dct.update({o: c for o, c in zip(options, candidates)}) - lst.append(dct) - - with Path(output_path).open("w") as handle: - json.dump(lst, handle, indent=4) - - @serialize - def load_label_studio_labels(self, input_path: str | Path) -> set[str]: - with Path(input_path).open("r") as handle: - data = json.load(handle) - - cpe_dset = self._prepare_cpe_dataset() - labeled_cert_digests: set[str] = set() - - logger.info("Translating label studio matches into their CPE representations and assigning to certificates.") - for annotation in tqdm(data, desc="Translating label studio matches"): - cpe_candidate_keys = { - key for key in annotation.keys() if "option_" in key and annotation[key] != "No good match" - } - - if "verified_cpe_match" not in annotation: - incorrect_keys: set[str] = set() - elif isinstance(annotation["verified_cpe_match"], str): - incorrect_keys = {annotation["verified_cpe_match"]} - else: - incorrect_keys = set(annotation["verified_cpe_match"]["choices"]) - - incorrect_keys = {x.lstrip("$") for x in incorrect_keys} - predicted_annotations = {annotation[x] for x in cpe_candidate_keys - incorrect_keys} - - cpes: set[CPE] = set() - for x in predicted_annotations: - if x not in cpe_dset.title_to_cpes: - logger.error(f"{x} not in dataset") - else: - to_update = cpe_dset.title_to_cpes[x] - if to_update and not cpes: - cpes = to_update - elif to_update and cpes: - cpes.update(to_update) - - # distinguish between FIPS and CC - if "\n" in annotation["text"]: - cert_name = annotation["text"].split("\nModule name: ")[1].split("\n")[0] - else: - cert_name = annotation["text"] - - certs = self._get_certs_by_name(cert_name) - labeled_cert_digests.update({x.dgst for x in certs}) - - for c in certs: - c.heuristics.verified_cpe_matches = {x.uri for x in cpes if x is not None} if cpes else None - - return labeled_cert_digests - - def enrich_automated_cpes_with_manual_labels(self) -> None: - """ - Prior to CVE matching, it is wise to expand the database of automatic CPE matches with those that were manually assigned. - """ - for cert in cast(Iterator[Certificate], self): - if not cert.heuristics.cpe_matches and cert.heuristics.verified_cpe_matches: - cert.heuristics.cpe_matches = cert.heuristics.verified_cpe_matches - elif cert.heuristics.cpe_matches and cert.heuristics.verified_cpe_matches: - cert.heuristics.cpe_matches = set(cert.heuristics.cpe_matches).union( - set(cert.heuristics.verified_cpe_matches) - ) - - @serialize - def compute_related_cves( - self, - download_fresh_cves: bool = False, - use_nist_cpe_matching_dict: bool = True, - ) -> None: - """ - Computes CVEs for the certificates, given their CPE matches. - """ - logger.info("Retrieving related CVEs to verified CPE matches") - if download_fresh_cves or not self.auxillary_datasets.cve_dset: - self.auxillary_datasets.cve_dset = self._prepare_cve_dataset( - download_fresh_cves, use_nist_cpe_matching_dict - ) - - logger.info("Computing heuristics: CVEs in certificates.") - self.enrich_automated_cpes_with_manual_labels() - cpe_rich_certs = [x for x in cast(Iterator[Certificate], self) if x.heuristics.cpe_matches] - - if not cpe_rich_certs: - logger.error( - "No certificates with verified CPE match detected. You must run dset.manually_verify_cpe_matches() first. Returning." - ) - return - - relevant_cpes = set(itertools.chain.from_iterable(x.heuristics.cpe_matches for x in cpe_rich_certs)) - self.auxillary_datasets.cve_dset.filter_related_cpes(relevant_cpes) - - cert: Certificate - for cert in tqdm(cpe_rich_certs, desc="Computing related CVES"): - if cert.heuristics.cpe_matches: - related_cves = [ - self.auxillary_datasets.cve_dset.get_cve_ids_for_cpe_uri(x) for x in cert.heuristics.cpe_matches - ] - related_cves = list(filter(lambda x: x is not None, related_cves)) - if related_cves: - cert.heuristics.related_cves = set( - itertools.chain.from_iterable(x for x in related_cves if x is not None) - ) - else: - cert.heuristics.related_cves = None - - n_vulnerable = len([x for x in cpe_rich_certs if x.heuristics.related_cves]) - n_vulnerabilities = sum([len(x.heuristics.related_cves) for x in cpe_rich_certs if x.heuristics.related_cves]) - logger.info( - f"In total, we identified {n_vulnerabilities} vulnerabilities in {n_vulnerable} vulnerable certificates." - ) - - def get_keywords_df(self, var: str) -> pd.DataFrame: - """ - Get dataframe of keyword hits for attribute (var) that is member of PdfData class. - """ - data = [dict({"dgst": x.dgst}, **x.pdf_data.get_keywords_df_data(var)) for x in self] - return pd.DataFrame(data).set_index("dgst") - - def update_with_certs(self, certs: list[CertSubType]) -> None: - """ - Enriches the dataset with `certs` - :param List[CommonCriteriaCert] certs: new certs to include into the dataset. - """ - if any([x not in self for x in certs]): - logger.warning("Updating dataset with certificates outside of the dataset!") - self.certs.update({x.dgst: x for x in certs}) diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py deleted file mode 100644 index bdf220c8..00000000 --- a/sec_certs/dataset/fips.py +++ /dev/null @@ -1,355 +0,0 @@ -from __future__ import annotations - -import datetime -import itertools -import logging -import shutil -from pathlib import Path -from typing import Final - -import numpy as np -import pandas as pd -from bs4 import BeautifulSoup, NavigableString - -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.dataset.cpe import CPEDataset -from sec_certs.dataset.cve import CVEDataset -from sec_certs.dataset.dataset import AuxillaryDatasets, Dataset -from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.model.reference_finder import ReferenceFinder -from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder -from sec_certs.sample.fips import FIPSCertificate -from sec_certs.serialization.json import ComplexSerializableType, serialize -from sec_certs.utils import helpers -from sec_certs.utils import parallel_processing as cert_processing -from sec_certs.utils.helpers import fips_dgst - -logger = logging.getLogger(__name__) - - -class FIPSAuxillaryDatasets(AuxillaryDatasets): - cpe_dset: CPEDataset | None = None - cve_dset: CVEDataset | None = None - algorithm_dset: FIPSAlgorithmDataset | None = None - - -class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxillaryDatasets], ComplexSerializableType): - """ - Class for processing of FIPSCertificate samples. Inherits from `ComplexSerializableType` and base abstract `Dataset` class. - """ - - def __init__( - self, - certs: dict[str, FIPSCertificate] = dict(), - root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH, - name: str | None = None, - description: str = None, - state: Dataset.DatasetInternalState | None = None, - auxillary_datasets: FIPSAuxillaryDatasets | None = None, - ): - self.certs = certs - self.timestamp = datetime.datetime.now() - self.sha256_digest = "not implemented" - self.name = name if name else type(self).__name__ + " dataset" - self.description = description if description else datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") - self.state = state if state else self.DatasetInternalState() - self.auxillary_datasets: FIPSAuxillaryDatasets = ( - auxillary_datasets if auxillary_datasets else FIPSAuxillaryDatasets() - ) - - self.root_dir = Path(root_dir) - - LIST_OF_CERTS_HTML: Final[dict[str, str]] = { - "fips_modules_active.html": constants.FIPS_ACTIVE_MODULES_URL, - "fips_modules_historical.html": constants.FIPS_HISTORICAL_MODULES_URL, - "fips_modules_revoked.html": constants.FIPS_REVOKED_MODULES_URL, - } - - @property - def policies_dir(self) -> Path: - return self.certs_dir / "policies" - - @property - def policies_pdf_dir(self) -> Path: - return self.policies_dir / "pdf" - - @property - def policies_txt_dir(self) -> Path: - return self.policies_dir / "txt" - - @property - def module_dir(self) -> Path: - return self.certs_dir / "modules" - - @property - def algorithm_dataset_path(self) -> Path: - return self.auxillary_datasets_dir / "algorithms.json" - - def __getitem__(self, item: str) -> FIPSCertificate: - try: - return super().__getitem__(item) - except KeyError: - return super().__getitem__(fips_dgst(item)) - - def _extract_data_from_html_modules(self) -> None: - """ - Extracts data from html module file - :param bool fresh: if all certs should be processed, or only the failed ones. Defaults to True - """ - logger.info("Extracting data from html modules.") - certs_to_process = [x for x in self if x.state.module_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - FIPSCertificate.parse_html_module, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting data from html modules", - ) - self.update_with_certs(processed_certs) - - @serialize - def extract_data(self) -> None: - logger.info("Extracting various data from certification artifacts.") - for cert in self: - cert.state.policy_extract_ok = True - cert.state.module_extract_ok = True - - self._extract_data_from_html_modules() - self._extract_policy_pdf_metadata() - self._extract_policy_pdf_keywords() - self._extract_algorithms_from_policy_tables() - - def _extract_policy_pdf_keywords(self) -> None: - logger.info("Extracting keywords from policy pdfs.") - certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - FIPSCertificate.extract_policy_pdf_keywords, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting keywords from policy pdfs", - ) - self.update_with_certs(processed_certs) - - def _download_all_artifacts_body(self, fresh: bool = True) -> None: - self._download_modules(fresh) - self._download_policies(fresh) - - def _download_modules(self, fresh: bool = True) -> None: - self.module_dir.mkdir(parents=True, exist_ok=True) - certs_to_process = [x for x in self if x.state.module_is_ok_to_download(fresh)] - - if fresh: - logger.info("Downloading HTML cryptographic modules.") - if not fresh and certs_to_process: - logger.info(f"Downloading {len(certs_to_process)} HTML modules for which download failed.") - - cert_processing.process_parallel( - FIPSCertificate.download_module, - certs_to_process, - config.n_threads, - progress_bar_desc="Downloading HTML modules", - ) - - def _download_policies(self, fresh: bool = True) -> None: - self.policies_pdf_dir.mkdir(parents=True, exist_ok=True) - certs_to_process = [x for x in self if x.state.policy_is_ok_to_download(fresh)] - - if fresh: - logger.info("Downloading PDF security policies.") - if not fresh and certs_to_process: - logger.info(f"Downloading {len(certs_to_process)} PDF security policies for which download failed.") - - cert_processing.process_parallel( - FIPSCertificate.download_policy, - certs_to_process, - config.n_threads, - progress_bar_desc="Downloading PDF security policies", - ) - - def _convert_all_pdfs_body(self, fresh: bool = True) -> None: - self._convert_policies_to_txt(fresh) - - def _convert_policies_to_txt(self, fresh: bool = True) -> None: - self.policies_txt_dir.mkdir(parents=True, exist_ok=True) - certs_to_process = [x for x in self if x.state.policy_is_ok_to_convert(fresh)] - - if fresh: - logger.info("Converting FIPS security policies to .txt") - if not fresh and certs_to_process: - logger.info( - f"Converting {len(certs_to_process)} FIPS security polcies to .txt for which previous convert failed." - ) - - cert_processing.process_parallel( - FIPSCertificate.convert_policy_pdf, - certs_to_process, - config.n_threads, - progress_bar_desc="Converting policies to pdf", - ) - - def _download_html_resources(self) -> None: - logger.info("Downloading HTML files that list FIPS certificates.") - html_urls = list(FIPSDataset.LIST_OF_CERTS_HTML.values()) - html_paths = [self.web_dir / x for x in FIPSDataset.LIST_OF_CERTS_HTML.keys()] - helpers.download_parallel(html_urls, html_paths) - - def _get_all_certs_from_html_sources(self) -> list[FIPSCertificate]: - return list( - itertools.chain.from_iterable( - self._get_certificates_from_html(self.web_dir / x) for x in self.LIST_OF_CERTS_HTML.keys() - ) - ) - - def _get_certificates_from_html(self, html_file: Path) -> list[FIPSCertificate]: - with open(html_file, encoding="utf-8") as handle: - html = BeautifulSoup(handle.read(), "html5lib") - - table = [x for x in html.find(id="searchResultsTable").tbody.contents if x != "\n"] - cert_ids: set[str] = set() - - for entry in table: - if isinstance(entry, NavigableString): - continue - cert_id = entry.find("a").text - if cert_id not in cert_ids: - cert_ids.add(cert_id) - - return [FIPSCertificate(cert_id) for cert_id in cert_ids] - - @classmethod - def from_web_latest(cls) -> FIPSDataset: - """ - Fetches the fresh snapshot of FIPSDataset from mirror. - """ - return cls.from_web(config.fips_latest_snapshot, "Downloading FIPS Dataset", "fips_latest_dataset.json") - - def _set_local_paths(self) -> None: - super()._set_local_paths() - if self.auxillary_datasets.algorithm_dset: - self.auxillary_datasets.algorithm_dset.json_path = self.algorithm_dataset_path - - cert: FIPSCertificate - for cert in self.certs.values(): - cert.set_local_paths(self.policies_pdf_dir, self.policies_txt_dir, self.module_dir) - - @serialize - def get_certs_from_web(self, to_download: bool = True, keep_metadata: bool = True) -> None: - self.web_dir.mkdir(parents=True, exist_ok=True) - - if to_download: - self._download_html_resources() - - logger.info("Adding unprocessed FIPS certificates into FIPSDataset.") - self.certs = {x.dgst: x for x in self._get_all_certs_from_html_sources()} - logger.info(f"The dataset now contains {len(self)} certificates.") - - if not keep_metadata: - shutil.rmtree(self.web_dir) - - self._set_local_paths() - self.state.meta_sources_parsed = True - - @serialize - def process_auxillary_datasets(self, download_fresh: bool = False) -> None: - super().process_auxillary_datasets(download_fresh) - self.auxillary_datasets.algorithm_dset = self._prepare_algorithm_dataset(download_fresh) - - def _prepare_algorithm_dataset(self, download_fresh_algs: bool = False) -> FIPSAlgorithmDataset: - logger.info("Preparing FIPSAlgorithm dataset.") - if not self.algorithm_dataset_path.exists() or download_fresh_algs: - alg_dset = FIPSAlgorithmDataset.from_web(self.algorithm_dataset_path) - alg_dset.to_json() - else: - alg_dset = FIPSAlgorithmDataset.from_json(self.algorithm_dataset_path) - - return alg_dset - - def _extract_algorithms_from_policy_tables(self): - logger.info("Extracting Algorithms from policy tables") - certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()] - cert_processing.process_parallel( - FIPSCertificate.get_algorithms_from_policy_tables, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting Algorithms from policy tables", - ) - - def _extract_policy_pdf_metadata(self) -> None: - logger.info("Extracting security policy metadata from the pdfs") - certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()] - processed_certs = cert_processing.process_parallel( - FIPSCertificate.extract_policy_pdf_metadata, - certs_to_process, - config.n_threads, - use_threading=False, - progress_bar_desc="Extracting security policy metadata", - ) - self.update_with_certs(processed_certs) - - def _compute_transitive_vulnerabilities(self) -> None: - logger.info("Computing heuristics: Computing transitive vulnerabilities in referenc(ed/ing) certificates.") - transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: cert.cert_id) - transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.policy_processed_references) - - for dgst in self.certs: - transitive_cve = transitive_cve_finder.predict_single_cert(dgst) - self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves - self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves - - def _prune_reference_candidates(self) -> None: - for cert in self: - cert.prune_referenced_cert_ids() - - # Previously, a following procedure was used to prune reference_candidates: - # - A set of algorithms was obtained via self.auxillary_datasets.algorithm_dset.get_algorithms_by_id(reference_candidate) - # - If any of these algorithms had the same vendor as the reference_candidate, the candidate was rejected - # - The rationale is that if an ID appears in a certificate s.t. an algorithm with the same ID was produced by the same vendor, the reference likely refers to alg. - # - Such reference should then be discarded. - # - We are uncertain of the effectivity of such measure, disabling it for now. - - def _compute_references(self, keep_unknowns: bool = False) -> None: - logger.info("Computing heuristics: Recovering references between certificates") - self._prune_reference_candidates() - - policy_reference_finder = ReferenceFinder() - policy_reference_finder.fit( - self.certs, lambda cert: cert.cert_id, lambda cert: cert.heuristics.policy_prunned_references - ) - - module_reference_finder = ReferenceFinder() - module_reference_finder.fit( - self.certs, lambda cert: cert.cert_id, lambda cert: cert.heuristics.module_prunned_references - ) - - for cert in self: - cert.heuristics.policy_processed_references = policy_reference_finder.predict_single_cert( - cert.dgst, keep_unknowns - ) - cert.heuristics.module_processed_references = module_reference_finder.predict_single_cert( - cert.dgst, keep_unknowns - ) - - def to_pandas(self) -> pd.DataFrame: - df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=FIPSCertificate.pandas_columns) - df = df.set_index("dgst") - - df.date_validation = pd.to_datetime(df.date_validation, infer_datetime_format=True) - df.date_sunset = pd.to_datetime(df.date_sunset, infer_datetime_format=True) - - # Manually delete one certificate with bad embodiment (seems to have many blank fields) - df = df.loc[~(df.embodiment == "*")] - - df = df.astype( - {"type": "category", "status": "category", "standard": "category", "embodiment": "category"} - ).fillna(value=np.nan) - - df.level = df.level.fillna(value=np.nan).astype("float") - # df.level = pd.Categorical(df.level, categories=sorted(df.level.dropna().unique().tolist()), ordered=True) - - # Introduce year when cert got valid - df["year_from"] = pd.DatetimeIndex(df.date_validation).year - - return df diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py deleted file mode 100644 index c48cff07..00000000 --- a/sec_certs/dataset/fips_algorithm.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -import itertools -import logging -import re -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Iterator - -import pandas as pd -from bs4 import BeautifulSoup - -from sec_certs import constants -from sec_certs.dataset.json_path_dataset import JSONPathDataset -from sec_certs.sample import FIPSAlgorithm -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.utils import helpers - -logger = logging.getLogger(__name__) - - -class FIPSAlgorithmDataset(JSONPathDataset, ComplexSerializableType): - def __init__( - self, algs: dict[str, FIPSAlgorithm] = dict(), json_path: str | Path = constants.DUMMY_NONEXISTING_PATH - ): - self.algs = algs - self.json_path = Path(json_path) - self.alg_number_to_algs: dict[str, set[FIPSAlgorithm]] = dict() - - self._build_lookup_dicts() - - @property - def serialized_attributes(self) -> list[str]: - return ["algs"] - - def __iter__(self) -> Iterator[FIPSAlgorithm]: - yield from self.algs.values() - - def __getitem__(self, item: str) -> FIPSAlgorithm: - return self.algs.__getitem__(item) - - def __setitem__(self, key: str, value: FIPSAlgorithm) -> None: - self.algs.__setitem__(key, value) - - def __len__(self) -> int: - return len(self.algs) - - def __contains__(self, item: FIPSAlgorithm) -> bool: - if not isinstance(item, FIPSAlgorithm): - raise ValueError(f"{item} is not of FIPSAlgorithm class") - return item.dgst in self.algs.keys() and self.algs[item.dgst] == item - - def __eq__(self, other: object) -> bool: - return isinstance(other, FIPSAlgorithmDataset) and self.algs == other.algs - - @classmethod - def from_web(cls, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> FIPSAlgorithmDataset: - with TemporaryDirectory() as tmp_dir: - htmls = FIPSAlgorithmDataset.download_alg_list_htmls(Path(tmp_dir)) - algs = set(itertools.chain.from_iterable(FIPSAlgorithmDataset.parse_algorithms_from_html(x) for x in htmls)) - return cls({x.dgst: x for x in algs}, json_path) - - @staticmethod - def download_alg_list_htmls(output_dir: Path) -> list[Path]: - first_page_path = output_dir / "page1.html" - ITEMS_PER_PAGE = "ipp=250" - - res = helpers.download_file(constants.FIPS_ALG_SEARCH_URL + "1&" + ITEMS_PER_PAGE, first_page_path) - if res != constants.RESPONSE_OK: - res = helpers.download_file(constants.FIPS_ALG_SEARCH_URL + "1&" + ITEMS_PER_PAGE, first_page_path) - if res != constants.RESPONSE_OK: - logger.error(f"Could not build Algorithm dataset, got server response: {res}") - raise ValueError(f"Could not build Algorithm dataset, got server response: {res}") - - n_pages = FIPSAlgorithmDataset.get_number_of_html_pages(first_page_path) - - urls = [constants.FIPS_ALG_SEARCH_URL + str(i) + "&" + ITEMS_PER_PAGE for i in range(2, n_pages + 1)] - paths = [output_dir / f"page{i}.html" for i in range(2, n_pages + 1)] - responses = helpers.download_parallel(urls, paths, progress_bar_desc="Downloading FIPS Algorithm HTMLs") - - failed_tuples = [ - (url, path) for url, path, resp in zip(urls, paths, responses) if resp != constants.RESPONSE_OK - ] - if failed_tuples: - failed_urls, failed_paths = zip(*failed_tuples) - responses = helpers.download_parallel(failed_urls, failed_paths) - if any([x != constants.RESPONSE_OK for x in responses]): - raise ValueError("Failed to download the algorithms HTML data, the dataset won't be constructed.") - - return paths - - @staticmethod - def get_number_of_html_pages(html_path: Path) -> int: - with html_path.open("r") as handle: - soup = BeautifulSoup(handle, "html5lib") - return int(soup.select("span[data-total-pages]")[0].attrs["data-total-pages"]) - - @staticmethod - def parse_algorithms_from_html(html_path: Path) -> set[FIPSAlgorithm]: - df = pd.read_html(html_path)[0] - df["alg_type"] = df["Validation Number"].map(lambda x: re.sub(r"[0-9\s]", "", x)) - df["alg_number"] = df["Validation Number"].map(lambda x: re.sub(r"[^0-9]", "", x)) - df["alg"] = df.apply( - lambda row: FIPSAlgorithm( - row["alg_number"], row["alg_type"], row["Vendor"], row["Implementation"], row["Validation Date"] - ), - axis=1, - ) - return set(df["alg"]) - - def to_pandas(self) -> pd.DataFrame: - df = pd.DataFrame([x.pandas_tuple for x in self], columns=FIPSAlgorithm.pandas_columns) - df = df.set_index("dgst") - return df - - def _build_lookup_dicts(self) -> None: - for alg in self: - if alg.alg_number not in self.alg_number_to_algs: - self.alg_number_to_algs[alg.alg_number] = {alg} - else: - self.alg_number_to_algs[alg.alg_number].add(alg) - - def get_algorithms_by_id(self, alg_number: str) -> set[FIPSAlgorithm]: - return self.alg_number_to_algs.get(alg_number, set()) diff --git a/sec_certs/dataset/fips_iut.py b/sec_certs/dataset/fips_iut.py deleted file mode 100644 index ce0f2f76..00000000 --- a/sec_certs/dataset/fips_iut.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Iterator, Mapping - -import requests - -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.dataset.dataset import logger -from sec_certs.dataset.json_path_dataset import JSONPathDataset -from sec_certs.sample.fips_iut import IUTSnapshot -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.utils.tqdm import tqdm - - -@dataclass -class IUTDataset(JSONPathDataset, ComplexSerializableType): - snapshots: list[IUTSnapshot] - _json_path: Path - - def __init__(self, snapshots: list[IUTSnapshot], json_path: str | Path = constants.DUMMY_NONEXISTING_PATH): - self.snapshots = snapshots - self.json_path = Path(json_path) - - def __iter__(self) -> Iterator[IUTSnapshot]: - yield from self.snapshots - - def __getitem__(self, item: int) -> IUTSnapshot: - return self.snapshots.__getitem__(item) - - def __len__(self) -> int: - return len(self.snapshots) - - @classmethod - def from_dumps(cls, dump_path: str | Path) -> IUTDataset: - directory = Path(dump_path) - fnames = list(directory.glob("*")) - snapshots = [] - for dump_path in tqdm(sorted(fnames), total=len(fnames)): - try: - snapshots.append(IUTSnapshot.from_dump(dump_path)) - except Exception as e: - logger.error(e) - return cls(snapshots) - - def to_dict(self) -> dict[str, list[IUTSnapshot]]: - return {"snapshots": list(self.snapshots)} - - @classmethod - def from_dict(cls, dct: Mapping) -> IUTDataset: - return cls(dct["snapshots"]) - - @classmethod - def from_web_latest(cls) -> IUTDataset: - """ - Get the IUTDataset from seccerts.org - """ - iut_resp = requests.get(config.fips_iut_dataset) - if iut_resp.status_code != 200: - raise ValueError(f"Getting IUT dataset failed: {iut_resp.status_code}") - with NamedTemporaryFile() as tmpfile: - tmpfile.write(iut_resp.content) - return cls.from_json(tmpfile.name) diff --git a/sec_certs/dataset/fips_mip.py b/sec_certs/dataset/fips_mip.py deleted file mode 100644 index 05ca5854..00000000 --- a/sec_certs/dataset/fips_mip.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Iterator, Mapping - -import requests - -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.dataset.dataset import logger -from sec_certs.dataset.json_path_dataset import JSONPathDataset -from sec_certs.sample.fips_mip import MIPSnapshot -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.utils.tqdm import tqdm - - -@dataclass -class MIPDataset(JSONPathDataset, ComplexSerializableType): - snapshots: list[MIPSnapshot] - _json_path: Path - - def __init__(self, snapshots: list[MIPSnapshot], json_path: str | Path = constants.DUMMY_NONEXISTING_PATH): - self.snapshots = snapshots - self.json_path = Path(json_path) - - def __iter__(self) -> Iterator[MIPSnapshot]: - yield from self.snapshots - - def __getitem__(self, item: int) -> MIPSnapshot: - return self.snapshots.__getitem__(item) - - def __len__(self) -> int: - return len(self.snapshots) - - @classmethod - def from_dumps(cls, dump_path: str | Path) -> MIPDataset: - directory = Path(dump_path) - fnames = list(directory.glob("*")) - snapshots = [] - for dump_path in tqdm(sorted(fnames), total=len(fnames)): - try: - snapshots.append(MIPSnapshot.from_dump(dump_path)) - except Exception as e: - logger.error(e) - return cls(snapshots) - - def to_dict(self) -> dict[str, list[MIPSnapshot]]: - return {"snapshots": list(self.snapshots)} - - @classmethod - def from_dict(cls, dct: Mapping) -> MIPDataset: - return cls(dct["snapshots"]) - - @classmethod - def from_web_latest(cls) -> MIPDataset: - """ - Get the MIPDataset from seccerts.org - """ - mip_resp = requests.get(config.fips_mip_dataset) - if mip_resp.status_code != 200: - raise ValueError(f"Getting MIP dataset failed: {mip_resp.status_code}") - with NamedTemporaryFile() as tmpfile: - tmpfile.write(mip_resp.content) - return cls.from_json(tmpfile.name) diff --git a/sec_certs/dataset/json_path_dataset.py b/sec_certs/dataset/json_path_dataset.py deleted file mode 100644 index 6eb4d968..00000000 --- a/sec_certs/dataset/json_path_dataset.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -import json -import logging -import shutil -from abc import ABC -from pathlib import Path - -from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, get_class_fullname - -logger = logging.getLogger(__name__) - - -class JSONPathDataset(ComplexSerializableType, ABC): - _json_path: Path - - @property - def json_path(self) -> Path: - return self._json_path - - @json_path.setter - def json_path(self, new_path: str | Path) -> None: - new_path = Path(new_path) - if new_path.is_dir(): - raise ValueError(f"Json path of {get_class_fullname(self)} cannot be a directory.") - - self._json_path = new_path - - def move_dataset(self, new_json_path: str | Path) -> None: - logger.info(f"Moving {get_class_fullname(self)} dataset to {new_json_path}") - new_json_path = Path(new_json_path) - new_json_path.parent.mkdir(parents=True, exist_ok=True) - - if self.json_path.exists(): - shutil.move(str(self.json_path), str(new_json_path)) - self.json_path = new_json_path - else: - self.json_path = new_json_path - self.to_json() - - @classmethod - def from_json(cls, input_path: str | Path): - with Path(input_path).open("r") as handle: - dset = json.load(handle, cls=CustomJSONDecoder) - dset.json_path = Path(input_path) - return dset diff --git a/sec_certs/dataset/protection_profile.py b/sec_certs/dataset/protection_profile.py deleted file mode 100644 index edfb6850..00000000 --- a/sec_certs/dataset/protection_profile.py +++ /dev/null @@ -1,98 +0,0 @@ -from __future__ import annotations - -import json -import logging -import shutil -import tempfile -from dataclasses import dataclass -from pathlib import Path - -import sec_certs.utils.helpers as helpers -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.serialization.json import get_class_fullname - -logger = logging.getLogger(__name__) - - -@dataclass -class ProtectionProfileDataset: - pps: dict[tuple[str, str | None], ProtectionProfile] - _json_path: Path - - def __init__( - self, - pps: dict[tuple[str, str | None], ProtectionProfile], - json_path: str | Path = constants.DUMMY_NONEXISTING_PATH, - ) -> None: - self.pps = pps - self.json_path = Path(json_path) - - @property - def json_path(self): - return self._json_path - - @json_path.setter - def json_path(self, new_path: str | Path): - new_path = Path(new_path) - if new_path.is_dir(): - raise ValueError(f"Json path of {get_class_fullname(self)} cannot be a directory.") - - self._json_path = new_path - - def move_dataset(self, new_json_path: str | Path) -> None: - logger.info(f"Moving {get_class_fullname(self)} dataset to {new_json_path}") - new_json_path = Path(new_json_path) - new_json_path.parent.mkdir(parents=True, exist_ok=True) - - if not self.json_path.exists(): - raise ValueError("Cannot move the PPDataset if the json path does not exist.") - - shutil.move(str(self.json_path), str(new_json_path)) - self.json_path = new_json_path - - def __iter__(self): - yield from self.pps.values() - - def __getitem__(self, item: tuple[str, str | None]) -> ProtectionProfile: - return self.pps.__getitem__(item) - - def __setitem__(self, key: tuple[str, str | None], value: ProtectionProfile): - self.pps.__setitem__(key, value) - - def __contains__(self, key): - return key in self.pps - - def __len__(self) -> int: - return len(self.pps) - - @classmethod - def from_json(cls, json_path: str | Path): - with Path(json_path).open("r") as handle: - data = json.load(handle) - pps = [ProtectionProfile.from_old_api_dict(x) for x in data.values()] - - dct = {} - for item in pps: - if (item.pp_name, item.pp_link) in dct: - logger.warning(f"Duplicate entry in PP dataset: {(item.pp_name, item.pp_link)}") - dct[(item.pp_name, item.pp_link)] = item - - return cls(dct) - - @classmethod - def from_web(cls, store_dataset_path: Path | None = None): - - logger.info(f"Downloading static PP dataset from: {config.pp_latest_snapshot}") - if not store_dataset_path: - tmp = tempfile.TemporaryDirectory() - store_dataset_path = Path(tmp.name) / "pp_dataset.json" - - helpers.download_file(config.pp_latest_snapshot, store_dataset_path) - obj = cls.from_json(store_dataset_path) - - if not store_dataset_path: - tmp.cleanup() - - return obj diff --git a/sec_certs/model/__init__.py b/sec_certs/model/__init__.py deleted file mode 100644 index fe1024f9..00000000 --- a/sec_certs/model/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -This package exposes model (mostly transformers and classifiers) that apply complex transformations. These are to be -leveraged by members of Dataset package and are directly applied on members of Sample class (or on built-in objects). -""" - -from sec_certs.model.cpe_matching import CPEClassifier -from sec_certs.model.reference_finder import ReferenceFinder -from sec_certs.model.sar_transformer import SARTransformer -from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder - -__all__ = ["CPEClassifier", "ReferenceFinder", "TransitiveVulnerabilityFinder", "SARTransformer"] diff --git a/sec_certs/model/cpe_matching.py b/sec_certs/model/cpe_matching.py deleted file mode 100644 index 0febea5d..00000000 --- a/sec_certs/model/cpe_matching.py +++ /dev/null @@ -1,404 +0,0 @@ -from __future__ import annotations - -import itertools -import logging -import operator -import re -from typing import Pattern - -import spacy -from rapidfuzz import fuzz -from sklearn.base import BaseEstimator - -from sec_certs import cert_rules, constants -from sec_certs.sample.cpe import CPE -from sec_certs.utils.tqdm import tqdm - -logger = logging.getLogger(__name__) - - -class CPEClassifier(BaseEstimator): - """ - Class that can predict CPE matches for certificate instances. - Adheres to sklearn BaseEstimator interface. - Fit method is called on list of CPEs and build two look-up dictionaries, see description of attributes. - """ - - vendor_to_versions_: dict[str, set[str]] # Key: CPE vendor, Value: versions of all CPE records of that vendor - vendor_version_to_cpe_: dict[tuple[str, str], set[CPE]] # Key: (CPE vendor, version), Value: CPEs (vendor, version) - vendors_: set[str] - - def __init__(self, match_threshold: int = 80, n_max_matches: int = 10, spacy_model_to_use: str = "en_core_web_sm"): - self.match_threshold = match_threshold - self.n_max_matches = n_max_matches - self.nlp = spacy.load(spacy_model_to_use, disable=["parser", "ner"]) - - def fit(self, X: list[CPE], y: list[str] | None = None) -> CPEClassifier: - """ - Just creates look-up structures from provided list of CPEs - - :param List[CPE] X: List of CPEs that can be matched with predict() - :param Optional[List[str]] y: will be ignored, specified to adhere to sklearn BaseEstimator interface, defaults to None - :return CPEClassifier: return self to allow method chaining - """ - self._build_lookup_structures(X) - return self - - @staticmethod - def _filter_short_cpes(cpes: list[CPE]) -> list[CPE]: - """ - Short CPE items are super easy to match with 100% rank, but they are hardly informative. This method discards them. - - :param List[CPE] cpes: List of CPEs to filtered - :return List[CPE]: All CPEs in cpes variable which item name has at least 4 characters. - """ - return list(filter(lambda x: x.item_name is not None and len(x.item_name) > 3, cpes)) - - def _build_lookup_structures(self, X: list[CPE]) -> None: - """ - Builds several look-up dictionaries for fast matching. - - vendor_to_version_: each vendor is mapped to set of versions that appear in combination with vendor in CPE dataset - - vendor_version_to_cpe_: Each (vendor, version) tuple is mapped to a set of CPE items that appear in combination with this tuple in CPE dataset - - vendors_: Just aggregates set of vendors, used for prunning later on. - - :param List[CPE] X: List of CPEs that will be used to build the dictionaries - """ - sufficiently_long_cpes = self._filter_short_cpes(X) - self.vendor_to_versions_ = {x.vendor: set() for x in sufficiently_long_cpes} - self.vendors_ = set(self.vendor_to_versions_.keys()) - self.vendor_version_to_cpe_ = dict() - - for cpe in tqdm(sufficiently_long_cpes, desc="Fitting the CPE classifier"): - self.vendor_to_versions_[cpe.vendor].add(cpe.version) - if (cpe.vendor, cpe.version) not in self.vendor_version_to_cpe_: - self.vendor_version_to_cpe_[(cpe.vendor, cpe.version)] = {cpe} - else: - self.vendor_version_to_cpe_[(cpe.vendor, cpe.version)].add(cpe) - - def predict(self, X: list[tuple[str, str, str]]) -> list[set[str] | None]: - """ - Will predict CPE uris for List of Tuples (vendor, product name, identified versions in product name) - - :param List[Tuple[str, str, str]] X: tuples (vendor, product name, identified versions in product name) - :return List[Optional[Set[str]]]: List of CPE uris that correspond to given input, None if nothing was found. - """ - return [self.predict_single_cert(x[0], x[1], x[2]) for x in tqdm(X, desc="Predicting")] - - def predict_single_cert( - self, - vendor: str | None, - product_name: str, - versions: set[str], - relax_version: bool = False, - relax_title: bool = False, - ) -> set[str] | None: - """ - Predict List of CPE uris for triplet (vendor, product_name, list_of_versions). The prediction is made as follows: - 1. Sanitize vendor name, lemmatize product name. - 2. Find vendors in CPE dataset that are related to the certificate - 3. Based on (vendors, versions) find all CPE items that are considered as candidates for match - 4. Compute string similarity of the candidate CPE matches and certificate name - 5. Evaluate best string similarity, if above threshold, declare it a match. - 6. If no CPE item is matched, try again but relax version and check CPEs that don't have their version specified. - 7. (Also, search for 100% CPE matches on item name instead of title.) - - :param Optional[str] vendor: manufacturer of the certificate - :param str product_name: name of the certificate - :param Set[str] versions: List of versions that appear in the certificate name - :param bool relax_version: See step 6 above., defaults to False - :param bool relax_title: See step 7 above, defaults to False - :return Optional[Set[str]]: Set of matching CPE uris, None if no matches found - """ - lemmatized_product_name = self._lemmatize_product_name(product_name) - candidate_vendors = self._get_candidate_list_of_vendors( - CPEClassifier._discard_trademark_symbols(vendor).lower() if vendor else vendor - ) - candidates = self._get_candidate_cpe_matches(candidate_vendors, versions) - candidates = self._filter_candidates_by_platform(candidates, product_name) - candidates = self._filter_candidates_by_update(candidates, lemmatized_product_name) - - ratings = [ - self._compute_best_match(cpe, lemmatized_product_name, candidate_vendors, versions, relax_title=relax_title) - for cpe in candidates - ] - threshold = self.match_threshold if not relax_version else 100 - final_matches_aux: list[tuple[float, CPE]] = list(filter(lambda x: x[0] >= threshold, zip(ratings, candidates))) - final_matches_aux = sorted(final_matches_aux, key=operator.itemgetter(0, 1), reverse=True) - final_matches: set[str] | None = { - x[1].uri for x in final_matches_aux[: self.n_max_matches] if x[1].uri is not None - } - - if not relax_title and not final_matches: - final_matches = self.predict_single_cert( - vendor, product_name, versions, relax_version=relax_version, relax_title=True - ) - - if not relax_version and not final_matches: - final_matches = self.predict_single_cert( - vendor, product_name, {constants.CPE_VERSION_NA}, relax_version=True, relax_title=relax_title - ) - - return final_matches if final_matches else None - - def _filter_candidates_by_update(self, cpes: list[CPE], cert_title: str) -> list[CPE]: - """ - Update means `service pack` or `release`. - """ - - def filter_condition(regex: Pattern, cpe: CPE, min_value: int, soft: bool = True): - if matches := re.findall(regex, cpe.update): - return int(re.findall(r"\d+", matches[0])[0]) >= min_value - return True if soft else False - - update_regexes = [cert_rules.SERVICE_PACK_RE, cert_rules.RELEASE_RE] - - for update_regex in update_regexes: - if matches := re.findall(update_regex, cert_title): - min_value = min([int(re.findall(r"\d+", x)[0]) for x in matches]) - soft = not any(re.search(update_regex, cpe.update + str(cpe.title)) for cpe in cpes) - return [x for x in cpes if filter_condition(update_regex, x, min_value, soft)] - - return cpes - - def _filter_candidates_by_platform(self, cpes: list[CPE], cert_title: str) -> list[CPE]: - def filter_condition(cpe: CPE, cert_platforms: set[str]): - if not cert_platforms and cpe.target_hw == "*": - return True - if cert_platforms and cpe.target_hw == "*": - return any(re.search(cert_rules.PLATFORM_REGEXES[x], str(cpe.title)) for x in cert_platforms) - if not cert_platforms and cpe.target_hw != "*": - return False - if cert_platforms and cpe.target_hw != "*": - target_hw_platforms = [ - platform - for platform, regex in cert_rules.PLATFORM_REGEXES.items() - if re.search(regex, cpe.target_hw) - ] - assert len(target_hw_platforms) <= 1 - can_return_true = any( - re.search(cert_rules.PLATFORM_REGEXES[x], cpe.target_hw + str(cpe.title)) for x in cert_platforms - ) - if not target_hw_platforms: - return can_return_true - else: - return can_return_true and target_hw_platforms[0] in cert_platforms - - crt_platforms = { - platform for platform, regex in cert_rules.PLATFORM_REGEXES.items() if re.search(regex, cert_title) - } - return [x for x in cpes if filter_condition(x, crt_platforms)] - - def _compute_best_match( - self, - cpe: CPE, - product_name: str, - candidate_vendors: set[str] | None, - versions: set[str], - relax_title: bool = False, - ) -> float: - """ - Tries several different settings in which string similarity between CPE and certificate name is tested. - For definition of string similarity, see rapidfuzz package on GitHub. Both token set ratio and partial ratio are tested, - always both on CPE title and CPE item name. - - :param CPE cpe: CPE to test - :param str product_name: name of the certificate - :param Optional[Set[str]] candidate_vendors: vendors that appear in the certificate - :param Set[str] versions: versions that appear in the certificate - :param bool relax_title: if to relax title or not, defaults to False - :return float: Maximal value of the four string similarities discussed above. - """ - if relax_title: - sanitized_title = ( - CPEClassifier._fully_sanitize_string(cpe.title) - if cpe.title - else CPEClassifier._fully_sanitize_string( - cpe.vendor + " " + cpe.item_name + " " + cpe.version + " " + cpe.update + " " + cpe.target_hw - ) - ) - else: - if cpe.title: - sanitized_title = CPEClassifier._fully_sanitize_string(cpe.title) - else: - return 0 - - # Sometimes, sanitization shortens CPE title to very short length. E.g., CPEs in Japanese unicode symbols that get all deteled. - if len(sanitized_title) < 5: - return 0 - - sanitized_item_name = CPEClassifier._fully_sanitize_string(cpe.item_name) - sanitized_cpe_stripped_manufacturer = re.sub(r"\b" + rf"{cpe.vendor}" + r"\b", "", sanitized_title) - standard_version_product_name = self._standardize_version_in_cert_name(product_name, versions) - - # The expression below is currently unused, it could assist with some matches though - # cert_stripped = CPEClassifier._strip_manufacturer_and_version(product_name, candidate_vendors, versions) - - # On some ratings, we require 100 match regardless of the treshold in settings. - ratings = [ - fuzz.token_set_ratio(product_name, sanitized_title), - fuzz.token_set_ratio(standard_version_product_name, sanitized_title), - fuzz.partial_token_sort_ratio(product_name, sanitized_title, score_cutoff=100), - fuzz.partial_token_sort_ratio(standard_version_product_name, sanitized_title, score_cutoff=100), - fuzz.partial_ratio(product_name, sanitized_title, score_cutoff=100), - fuzz.partial_ratio(standard_version_product_name, sanitized_title, score_cutoff=100), - ] - - # Big-IP has dumb CPEs that contain only that string in item name, which leads to false positives. - if relax_title and cpe.item_name != "big-ip": - ratings += [ - fuzz.token_set_ratio(product_name, sanitized_cpe_stripped_manufacturer, score_cutoff=100), - fuzz.partial_ratio(product_name, sanitized_cpe_stripped_manufacturer, score_cutoff=100), - fuzz.token_set_ratio(product_name, sanitized_item_name, score_cutoff=100), - fuzz.partial_ratio(product_name, sanitized_item_name, score_cutoff=100), - ] - - return max(ratings) - - @staticmethod - def _fully_sanitize_string(string: str) -> str: - return CPEClassifier._replace_special_chars_with_space( - CPEClassifier._discard_trademark_symbols(string.lower()) - ).strip() - - @staticmethod - def _replace_special_chars_with_space(string: str) -> str: - return re.sub(r"[^a-zA-Z0-9 \n\.]", " ", string) - - @staticmethod - def _discard_trademark_symbols(string: str) -> str: - return string.replace("®", "").replace("™", "") - - @staticmethod - def _strip_manufacturer_and_version(string: str, manufacturers: set[str] | None, versions: set[str]) -> str: - to_strip = versions | manufacturers if manufacturers else versions - for x in to_strip: - string = string.lower().replace(CPEClassifier._replace_special_chars_with_space(x.lower()), " ").strip() - return string - - @staticmethod - def _standardize_version_in_cert_name(string: str, detected_versions: set[str]) -> str: - for ver in detected_versions: - version_regex = r"(" + r"(\bversion)\s*" + ver + r"+) | (\bv\s*" + ver + r"+)" - string = re.sub(version_regex, " " + ver + " ", string, flags=re.IGNORECASE) - return string - - def _process_manufacturer(self, manufacturer: str, result: set) -> set[str]: - tokenized = manufacturer.split() - if tokenized[0] in self.vendors_: - result.add(tokenized[0]) - if len(tokenized) > 1 and tokenized[0] + tokenized[1] in self.vendors_: - result.add(tokenized[0] + tokenized[1]) - - # Below are completely manual fixes - if "hewlett" in tokenized or "hewlett-packard" in tokenized or manufacturer == "hewlett packard": - result.add("hp") - if "thales" in tokenized: - result.add("thalesesecurity") - result.add("thalesgroup") - if "stmicroelectronics" in tokenized: - result.add("st") - if "athena" in tokenized and "smartcard" in tokenized: - result.add("athena-scs") - if tokenized[0] == "the" and not result: - candidate_result = self._get_candidate_list_of_vendors(" ".join(tokenized[1:])) - return set(candidate_result) if candidate_result else set() - - return set(result) if result else set() - - def _get_candidate_list_of_vendors(self, manufacturer: str | None) -> set[str]: - """ - Given manufacturer name, this method will find list of plausible vendors from CPE dataset that are likely related. - - :param Optional[str] manufacturer: manufacturer - :return Set[str]: List of related manufacturers, None if nothing relevant is found. - """ - result: set[str] = set() - if not manufacturer: - return result - - splits = re.compile(r"[,/]").findall(manufacturer) - - if splits: - vendor_tokens = set( - itertools.chain.from_iterable([x.strip() for x in manufacturer.split(s)] for s in splits) - ) - result_aux = [self._get_candidate_list_of_vendors(x) for x in vendor_tokens] - result_used = set(set(itertools.chain.from_iterable(x for x in result_aux if x))) - return result_used if result_used else set() - - if manufacturer in self.vendors_: - result.add(manufacturer) - - return self._process_manufacturer(manufacturer, result) - - def _get_candidate_vendor_version_pairs( - self, cert_candidate_cpe_vendors: set[str], cert_candidate_versions: set[str] - ) -> list[tuple[str, str]] | None: - """ - Given parameters, will return Pairs (cpe_vendor, cpe_version) that are relevant to a given sample - - - :param Set[str] cert_candidate_cpe_vendors: list of CPE vendors relevant to a sample - :param Set[str] cert_candidate_versions: List of versions heuristically extracted from the sample name - :return Optional[List[Tuple[str, str]]]: List of tuples (cpe_vendor, cpe_version) that can be used in the lookup table to search the CPE dataset. - """ - - def is_cpe_version_among_cert_versions(cpe_version: str | None, cert_versions: set[str]) -> bool: - def simple_startswith(seeked_version: str, checked_string: str) -> bool: - if seeked_version == checked_string: - return True - else: - return ( - checked_string.startswith(seeked_version) and not checked_string[len(seeked_version)].isdigit() - ) - - if not cpe_version: - return False - just_numbers = r"(\d{1,5})(\.\d{1,5})" - - # This assures that on cert version with at least two tokens, we don't match only one-token CPE. - # E.g. cert with version 7.6 must not match CPE record of version 7 - if len(cert_versions) == 1 and len(list(cert_versions)[0]) >= 3 and len(cpe_version) < 3: - return False - - # Except from startswith stuff, this also mandates that for long enough cert vesions (e.g. `3.1`) we do not - # match too short CPE versions, e.g. `3` - for v in cert_versions: - if ( - (simple_startswith(v, cpe_version) and re.search(just_numbers, cpe_version)) - or simple_startswith(cpe_version, v) - ) and (len(v) < 3 or len(cpe_version) >= 3): - return True - return False - - if not cert_candidate_cpe_vendors: - return None - - candidate_vendor_version_pairs: list[tuple[str, str]] = [] - for vendor in cert_candidate_cpe_vendors: - viable_cpe_versions = self.vendor_to_versions_.get(vendor, set()) - matched_cpe_versions = [ - x for x in viable_cpe_versions if is_cpe_version_among_cert_versions(x, cert_candidate_versions) - ] - candidate_vendor_version_pairs.extend([(vendor, x) for x in matched_cpe_versions]) - return candidate_vendor_version_pairs - - def _get_candidate_cpe_matches(self, candidate_vendors: set[str], candidate_versions: set[str]) -> list[CPE]: - """ - Given List of candidate vendors and candidate versions found in certificate, candidate CPE matches are found - - :param Set[str] candidate_vendors: List of version strings that were found in the certificate - :param Set[str] candidate_versions: List of vendor strings that were found in the certificate - :return List[CPE]: List of CPE records that could match, to be refined later - """ - candidate_vendor_version_pairs = self._get_candidate_vendor_version_pairs(candidate_vendors, candidate_versions) - return ( - list(itertools.chain.from_iterable(self.vendor_version_to_cpe_[x] for x in candidate_vendor_version_pairs)) - if candidate_vendor_version_pairs - else [] - ) - - def _lemmatize_product_name(self, product_name: str) -> str: - if not product_name: - return product_name - return " ".join([token.lemma_ for token in self.nlp(CPEClassifier._fully_sanitize_string(product_name))]) diff --git a/sec_certs/model/evaluation.py b/sec_certs/model/evaluation.py deleted file mode 100644 index dd5eaffd..00000000 --- a/sec_certs/model/evaluation.py +++ /dev/null @@ -1,101 +0,0 @@ -from __future__ import annotations - -import json -import logging -from pathlib import Path - -import numpy as np - -import sec_certs.utils.helpers as helpers -from sec_certs.dataset.cpe import CPEDataset -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.fips import FIPSCertificate -from sec_certs.serialization.json import CustomJSONEncoder - -logger = logging.getLogger(__name__) - - -def get_validation_dgsts(filepath: str | Path) -> set[str]: - with Path(filepath).open("r") as handle: - return set(json.load(handle)) - - -def compute_precision(y: np.ndarray, y_pred: np.ndarray, **kwargs) -> float: - prec = [] - for true, pred in zip(y, y_pred): - set_pred = set(pred) if pred else set() - set_true = set(true) if true else set() - if not set_pred and not set_true: - pass - else: - prec.append(len(set_true) / len(set_true.union(set_pred))) - return np.mean(prec) # type: ignore - - -def evaluate( - x_valid: list[CommonCriteriaCert | FIPSCertificate], - y_valid: list[set[str] | None], - outpath: Path | str | None, - cpe_dset: CPEDataset, -) -> None: - y_pred = [x.heuristics.cpe_matches for x in x_valid] - precision = compute_precision(np.array(y_valid), np.array(y_pred)) - - correctly_classified = [] - badly_classified = [] - n_cpes_lost = 0 - n_certs_has_lost_cpes = 0 - n_certs_lost = 0 - - for cert, predicted_cpes, verified_cpes in zip(x_valid, y_pred, y_valid): - if not verified_cpes: - verified_cpes = set() - verified_cpes_dict = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in verified_cpes} - - if not predicted_cpes: - predicted_cpes = set() - predicted_cpes_dict = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in predicted_cpes} - - cert_name = cert.name if isinstance(cert, CommonCriteriaCert) else cert.web_data.module_name - vendor = cert.manufacturer if isinstance(cert, CommonCriteriaCert) else cert.web_data.vendor - - should_be_removed = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in predicted_cpes - verified_cpes} - should_be_added = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in verified_cpes - predicted_cpes} - - record = { - "certificate_name": cert_name, - "vendor": vendor, - "heuristic version": helpers.compute_heuristics_version(cert_name) if cert_name else None, - "predicted_cpes": predicted_cpes_dict, - "manually_assigned_cpes": verified_cpes_dict, - "should_be_removed": should_be_removed, - "should_be_added": should_be_added, - } - - if not predicted_cpes and verified_cpes: - n_certs_lost += 1 - - if predicted_cpes.issubset(verified_cpes): - correctly_classified.append(record) - else: - badly_classified.append(record) - - if should_be_added: - n_certs_has_lost_cpes += 1 - n_cpes_lost += len(should_be_added) - - results = { - "Precision": precision, - "n_cpes_lost": n_cpes_lost, - "n_certs_has_lost_cpes": n_certs_has_lost_cpes, - "n_certs_lost": n_certs_lost, - "correctly_classified": correctly_classified, - "badly_classified": badly_classified, - } - logger.info( - f"Precision: {precision}; the classifier now misses {n_cpes_lost} CPE matches from {n_certs_has_lost_cpes} certificates ({n_certs_lost} certificates no longer have a match). In total, {sum([len(x) for x in y_pred if x])} CPEs were matched in {len(y_pred)} certs." - ) - - if outpath: - with Path(outpath).open("w") as handle: - json.dump(results, handle, indent=4, cls=CustomJSONEncoder, sort_keys=True) diff --git a/sec_certs/model/reference_finder.py b/sec_certs/model/reference_finder.py deleted file mode 100644 index 117a8b9d..00000000 --- a/sec_certs/model/reference_finder.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -from typing import Callable, Dict, List, Optional, Set, TypeVar - -from sec_certs.sample.certificate import Certificate, References - -CertSubType = TypeVar("CertSubType", bound=Certificate) -Certificates = Dict[str, CertSubType] -ReferencedByDirect = Dict[str, Set[str]] -ReferencedByIndirect = Dict[str, Set[str]] -ReferencesType = Dict[str, Dict[str, Optional[Set[str]]]] -IDMapping = Dict[str, List[str]] -UnknownReferences = Dict[str, Set[str]] -IDLookupFunc = Callable[[CertSubType], str] -ReferenceLookupFunc = Callable[[CertSubType], Set[str]] - - -# TODO: All of this can and should be rewritten on top of networkx or some other graph library. -class ReferenceFinder: - """ - The class assigns references of other certificate instances for each instance. - Adheres to sklearn BaseEstimator interface. - The fit is called on a dictionary of certificates, builds a hashmap of references, and assigns references for each certificate in the dictionary. - """ - - def __init__(self): - self.references: ReferencesType = {} - self.id_mapping: IDMapping = {} - self._fitted: bool = False - - def _create_id_mapping(self, certificates: Certificates, id_func: IDLookupFunc) -> None: - """ - Create the ID mapping of certificate IDs to certificate digests. - - Necessary for handling duplicates. - """ - # Create a mapping of certificate ID to certificate digests with that ID. - for dgst in certificates: - cert_id = id_func(certificates[dgst]) - c_list = self.id_mapping.setdefault(cert_id, []) - c_list.append(dgst) - - # Sort digests in ID mapping to have deterministic behavior. - # The certificate with the first digest will be used with that ID, others will be discarded. - for digests in self.id_mapping.values(): - digests.sort() - - def _compute_indirect_references(self, referenced_by: ReferencedByDirect) -> ReferencedByIndirect: - """ - Compute indirect references via a BFS algorithm. - """ - referenced_by_indirect: ReferencedByIndirect = {} - - # Populate with direct references. - certs_id_list = referenced_by.keys() - for cert_id in certs_id_list: - referenced_by_indirect[cert_id] = set() - for item in referenced_by[cert_id]: - referenced_by_indirect[cert_id].add(item) - - # Flood in the indirect ones. - new_change_detected = True - while new_change_detected: - new_change_detected = False - - for cert_id in certs_id_list: - tmp_referenced_by_indirect_nums = referenced_by_indirect[cert_id].copy() - for referencing in tmp_referenced_by_indirect_nums: - if referencing in certs_id_list: - tmp_referencing = referenced_by_indirect[referencing].copy() - newly_discovered_references = [ - x for x in tmp_referencing if x not in referenced_by_indirect[cert_id] - ] - referenced_by_indirect[cert_id].update(newly_discovered_references) - if newly_discovered_references: - new_change_detected = True - return referenced_by_indirect - - def _build_referenced_by( - self, certificates: Certificates, ref_lookup_func: ReferenceLookupFunc - ) -> tuple[ReferencedByDirect, ReferencedByIndirect]: - referenced_by: ReferencedByDirect = {} - - for this_cert_id, cert_digests in self.id_mapping.items(): - # Take the first certificate digest from the ID mapping (to ensure deterministic behavior and resolve duplicates). - # TODO: A better approach for handling duplicates in the future would be nice. - cert_dgst = cert_digests[0] - cert_obj = certificates[cert_dgst] - - refs = ref_lookup_func(cert_obj) - if refs is None: - continue - - # Process direct reference - # All are added here, the unknown ones are filtered later on. - for cert_id in refs: - if cert_id == this_cert_id: - continue - referenced_by.setdefault(cert_id, set()) - referenced_by[cert_id].add(this_cert_id) - - # Now do the indirect ones - referenced_by_indirect = self._compute_indirect_references(referenced_by) - return referenced_by, referenced_by_indirect - - def _get_reverse_references( - self, cert_id: str, references: ReferencedByDirect | ReferencedByIndirect - ) -> set[str] | None: - result = set() - - for other_id in references: - if cert_id in references[other_id]: - result.add(other_id) - - return result if result else None - - def _build_referencing( - self, referenced_by_direct: ReferencedByDirect, referenced_by_indirect: ReferencedByIndirect - ) -> None: - for cert_id, cert_digests in self.id_mapping.items(): - cert_dgst = cert_digests[0] - self.references[cert_dgst] = { - "directly_referenced_by": referenced_by_direct.get(cert_id, None), - "indirectly_referenced_by": referenced_by_indirect.get(cert_id, None), - "directly_referencing": self._get_reverse_references(cert_id, referenced_by_direct), - "indirectly_referencing": self._get_reverse_references(cert_id, referenced_by_indirect), - } - - def fit(self, certificates: Certificates, id_func: IDLookupFunc, ref_lookup_func: ReferenceLookupFunc) -> None: - """ - Builds a list of references and assigns references for each certificate instance. - - :param Certificates certificates: dictionary of certificates with hashes as key - :param IDLookupFunc id_func: lookup function for cert id - :param ReferenceLookupFunc ref_lookup_func: lookup for references - """ - if self._fitted: - raise ValueError("Finder already fitted") - # Create the ID mapping first so that we can resolve duplicates. - self._create_id_mapping(certificates, id_func) - - # Build the referenced_by first - referenced_by_direct, referenced_by_indirect = self._build_referenced_by(certificates, ref_lookup_func) - - # Build the referencing second (this actually writes into self.references). - self._build_referencing(referenced_by_direct, referenced_by_indirect) - self._fitted = True - - @property - def unknown_references(self) -> UnknownReferences: - """ - Get the unknown references in the fitted dataset (to unknown certificate IDs, not in the dataset during fit). - """ - if not self._fitted: - return {} - result = {} - for cert_id, digests in self.id_mapping.items(): - cert_digest = digests[0] - cert_references = self.references[cert_digest] - direct_refs = cert_references["directly_referencing"] - if not direct_refs: - continue - unknowns = set(filter(lambda refd_cert_id: refd_cert_id not in self.id_mapping, direct_refs)) - if unknowns: - result[cert_id] = unknowns - return result - - @property - def duplicates(self) -> IDMapping: - """ - Get the duplicates in the fitted dataset. - - :return IDMapping: Mapping of certificate ID to digests that share it. - """ - if not self._fitted: - return {} - return {cert_id: digests for cert_id, digests in self.id_mapping.items() if len(digests) > 1} - - def predict_single_cert(self, dgst: str, keep_unknowns: bool = True) -> References: - """ - Get the references object for specified certificate digest. - - :param dgst: certificate digest - :param keep_unknowns: Whether to keep references to unknown certificate IDs - :return References: References object - """ - if not self._fitted: - raise ValueError("Finder not yet fitted") - - def wrap(res): - if not res: - return None - # If we do not want the unknown references, filter them here. - if not keep_unknowns: - res = set(filter(lambda cert_id: cert_id in self.id_mapping, res)) - return set(res) if res else None - - if dgst not in self.references: - return References() - - return References( - wrap(self.references[dgst].get("directly_referenced_by", None)), - wrap(self.references[dgst].get("indirectly_referenced_by", None)), - wrap(self.references[dgst].get("directly_referencing", None)), - wrap(self.references[dgst].get("indirectly_referencing", None)), - ) - - def predict(self, dgst_list: list[str], keep_unknowns: bool = True) -> dict[str, References]: - """ - Get the references for a list of certificate digests. - - :param dgst_list: List of certificate digests. - :param keep_unknowns: Whether to keep references to and from unknown certificate IDs - :return Dict[str, References]: Dict with certificate hash and References object. - """ - if not self._fitted: - raise ValueError("Finder not yet fitted") - cert_references = {} - - for dgst in dgst_list: - cert_references[dgst] = self.predict_single_cert(dgst, keep_unknowns=keep_unknowns) - - return cert_references diff --git a/sec_certs/model/sar_transformer.py b/sec_certs/model/sar_transformer.py deleted file mode 100644 index a5bcff57..00000000 --- a/sec_certs/model/sar_transformer.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Dict, Iterable, cast - -from sklearn.base import BaseEstimator, TransformerMixin - -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.sar import SAR, SAR_DICT_KEY - -logger = logging.getLogger(__name__) - - -# TODO: Right now we ignore number of ocurrences for final SAR selection. If we keep it this way, we can discard that variable -class SARTransformer(BaseEstimator, TransformerMixin): - """ - Class for transforming SARs defined in st_keywords and report_keywords dictionaries into SAR objects. - This class implements sklearn transformer interface, so fit_transform() can be called on it. - """ - - def fit(self, certificates: Iterable[CommonCriteriaCert]) -> SARTransformer: - """ - Just returns self, no fitting needed - - :param Iterable[CommonCriteriaCert] certificates: Unused parameter - :return SARTransformer: return self - """ - return self - - def transform(self, certificates: Iterable[CommonCriteriaCert]) -> list[set[SAR] | None]: - """ - Just a wrapper around transform_single_cert() called on an iterable of CommonCriteriaCert. - - :param Iterable[CommonCriteriaCert] certificates: Iterable of CommonCriteriaCert objects to perform the extraction on. - :return List[Optional[Set[SAR]]]: Returns List of results from transform_single_cert(). - """ - return [self.transform_single_cert(cert) for cert in certificates] - - def transform_single_cert(self, cert: CommonCriteriaCert) -> set[SAR] | None: - """ - Given CommonCriteriaCert, will transform SAR keywords extracted from txt files - into a set of SAR objects. Also handles extractin of correct SAR levels, duplicities and filtering. - Uses three sources: CSV scan, security target, and certification report. - The caller should assure that the certificates have the keywords extracted. - - :param CommonCriteriaCert cert: Certificate to extract SARs from - :return Optional[Set[SAR]]: Set of SARs, None if none were identified. - """ - sec_level_candidates, st_candidates, report_candidates = self._collect_sar_candidates_from_all_sources(cert) - return self._resolve_candidate_conflicts(sec_level_candidates, st_candidates, report_candidates, cert.dgst) - - @staticmethod - def _collect_sar_candidates_from_all_sources(cert: CommonCriteriaCert) -> tuple[set[SAR], set[SAR], set[SAR]]: - """ - Parses SARs from three distinct sources and returns the results as a three tuple: - - Security level from CSV scan - - Keywords from Security target - - Keywords from Certification report - """ - - def st_keywords_may_have_sars(sample: CommonCriteriaCert): - return sample.pdf_data.st_keywords and SAR_DICT_KEY in sample.pdf_data.st_keywords - - def report_keywords_may_have_sars(sample: CommonCriteriaCert): - return sample.pdf_data.report_keywords and SAR_DICT_KEY in sample.pdf_data.report_keywords - - sec_level_sars = SARTransformer._parse_sars_from_security_level_list(cert.security_level) - - if st_keywords_may_have_sars(cert): - st_dict: dict = cast(Dict, cert.pdf_data.st_keywords) - st_sars = SARTransformer._parse_sar_dict(st_dict[SAR_DICT_KEY], cert.dgst) - else: - st_sars = set() - - if report_keywords_may_have_sars(cert): - report_dict: dict = cast(Dict, cert.pdf_data.report_keywords) - report_sars = SARTransformer._parse_sar_dict(report_dict[SAR_DICT_KEY], cert.dgst) - else: - report_sars = set() - - return sec_level_sars, st_sars, report_sars - - @staticmethod - def _resolve_candidate_conflicts( - sec_level_candidates: set[SAR], st_candidates: set[SAR], report_candidates: set[SAR], cert_dgst: str - ) -> set[SAR] | None: - final_candidates: dict[str, SAR] = {x.family: x for x in sec_level_candidates} - """ - Given three parameters (SAR candidates from csv scan, ST and cert. report), builds final list of SARs in cert. - This is done as follows: - - All SARs from security level are added first - - Non-conflicting SARs from security target are added as well - - Non-conflicting SARs from ceritifcation report are added at last - - Note: Conflict means an attempt to add SAR of family (but different level) that is already present in the set. - - :return Set | None: Returns set of SARs or None if empty - """ - for candidate in st_candidates: - if candidate.family in final_candidates and candidate != final_candidates[candidate.family]: - logger.debug( - f"Cert {cert_dgst} SAR conflict: Attempting to add {candidate} from ST, conflicts with {final_candidates[candidate.family]}" - ) - else: - final_candidates[candidate.family] = candidate - - for candidate in report_candidates: - if candidate.family in final_candidates and candidate != final_candidates[candidate.family]: - logger.debug( - f"Cert {cert_dgst} SAR conflict: Attempting to add {candidate} from cert. report, conflicts with {final_candidates[candidate.family]}" - ) - else: - final_candidates[candidate.family] = candidate - - return set(final_candidates.values()) if final_candidates else None - - @staticmethod - def _parse_sar_dict(dct: dict[str, dict[str, int]], dgst: str) -> set[SAR]: - """ - Accepts st_keywords or report_keywords dictionary. Will reconstruct SAR objects from it. Each SAR family can - appear multiple times in the dictionary (due to conflicts) with different levels. Iterated item will replace - existing record if: - - it has higher level than than the current SAR - - Only SARs with recovered level are considered, e.g. ASE_REQ.2 is valid string while ASE_REQ is not. - - :param dct: _description_ - :param dgst: DIgest of the processed certificate. - :return: _description_ - """ - sars: dict[str, tuple[SAR, int]] = dict() - for sar_class, class_matches in dct.items(): - for sar_string, n_occurences in class_matches.items(): - try: - candidate = SAR.from_string(sar_string) - except ValueError as e: - logger.debug(f"Badly formatted SAR string {sar_string}, skipping: {e}") - continue - - if candidate.family in sars: - logger.debug( - f"Cert {dgst} Attempting to add {candidate} while {sars[candidate.family]} already in SARS" - ) - - if (candidate.family not in sars) or ( - candidate.family in sars and candidate.level > sars[candidate.family][0].level - ): - sars[candidate.family] = (candidate, n_occurences) - return {x[0] for x in sars.values()} if sars else set() - - @staticmethod - def _parse_sars_from_security_level_list(lst: Iterable[str]) -> set[SAR]: - sars = set() - for element in lst: - try: - sars.add(SAR.from_string(element)) - except ValueError: - continue - return sars diff --git a/sec_certs/model/transitive_vulnerability_finder.py b/sec_certs/model/transitive_vulnerability_finder.py deleted file mode 100644 index 1d4c8243..00000000 --- a/sec_certs/model/transitive_vulnerability_finder.py +++ /dev/null @@ -1,149 +0,0 @@ -from __future__ import annotations - -import logging -from collections import Counter -from dataclasses import dataclass, field -from enum import Enum -from typing import Callable, Dict, Optional, Set, TypeVar - -from sec_certs.sample.certificate import Certificate, References -from sec_certs.serialization.json import ComplexSerializableType - -CertSubType = TypeVar("CertSubType", bound=Certificate) -IDLookupFunc = Callable[[CertSubType], str] -ReferenceLookupFunc = Callable[[CertSubType], References] - - -class ReferenceType(Enum): - DIRECT = "direct" - INDIRECT = "indirect" - - -@dataclass -class TransitiveCVEs(ComplexSerializableType): - direct_transitive_cves: set[str] | None = field(default=None) - indirect_transitive_cves: set[str] | None = field(default=None) - - -Certificates = Dict[str, CertSubType] -Vulnerabilities = Dict[str, Dict[str, Optional[Set[str]]]] - - -class TransitiveVulnerabilityFinder: - """ - The class assigns vulnerabilities to each certificate instance caused by references among certificate instances. - Adheres to sklearn BaseEstimator interface. - """ - - def __init__(self, id_func: IDLookupFunc): - self.vulnerabilities: Vulnerabilities = {} - self.certificates: Certificates = {} - self._cert_id_counter: Counter = Counter() - self._fitted = False - self._id_func: IDLookupFunc = id_func - - def _clear_state(self) -> None: - self.vulnerabilities = {} - self.certificates = {} - self._cert_id_counter = Counter() - - def _fill_dataset_cert_ids_counter(self) -> None: - self._cert_id_counter = Counter([self._id_func(x) for x in self.certificates.values()]) - - def _get_cert_transitive_cves( - self, cert: CertSubType, reference_type: ReferenceType, ref_func: ReferenceLookupFunc - ) -> set[str] | None: - - references = ( - ref_func(cert).directly_referenced_by - if reference_type == ReferenceType.DIRECT - else ref_func(cert).indirectly_referenced_by - ) - - if not references: - return None - - vulnerabilities = set() - - for cert_id in references: - if self._cert_id_counter[cert_id] != 1: - continue - - for cert in self.certificates.values(): - cves = cert.heuristics.related_cves - if self._id_func(cert) == cert_id and cves: - vulnerabilities.update(cves) - - return vulnerabilities if vulnerabilities else None - - def fit(self, certificates: Certificates, ref_func: ReferenceLookupFunc) -> Vulnerabilities: - """ - Method assigns each certificate vulnerabilities caused by references among certificates - - :param Certificates certificates: Dictionary of certificates with digests - :return Vulnerabilities: Dictionary of vulnerabilities of certificate instances - """ - self._clear_state() - self.certificates = certificates - self._fill_dataset_cert_ids_counter() - - thrown_away_cert_counter = 0 - - for cert in self.certificates.values(): - cert_id = self._id_func(cert) - - if not cert_id: - continue - if self._cert_id_counter[cert_id] > 1: - thrown_away_cert_counter += 1 - continue - - self.vulnerabilities[cert.dgst] = dict() - self.vulnerabilities[cert.dgst][ReferenceType.DIRECT.value] = self._get_cert_transitive_cves( - cert, ReferenceType.DIRECT, ref_func - ) - self.vulnerabilities[cert.dgst][ReferenceType.INDIRECT.value] = self._get_cert_transitive_cves( - cert, ReferenceType.INDIRECT, ref_func - ) - - if thrown_away_cert_counter > 0: - logging.warning("There were total of %s certificates skipped due to duplicity", thrown_away_cert_counter) - - self._fitted = True - - return self.vulnerabilities - - def predict_single_cert(self, dgst: str) -> TransitiveCVEs: - """ - Method returns vulnerabilities for certificate digest - - :param str dgst: Digest of certificate - :return TransitiveCVE: TransitiveCVE object of certificate - """ - if not self._fitted: - raise ValueError("Finder not yet fitted") - - if not self.vulnerabilities.get(dgst): - return TransitiveCVEs(direct_transitive_cves=None, indirect_transitive_cves=None) - - return TransitiveCVEs( - self.vulnerabilities[dgst][ReferenceType.DIRECT.value], - self.vulnerabilities[dgst][ReferenceType.INDIRECT.value], - ) - - def predict(self, dgst_list: list[str]) -> dict[str, TransitiveCVEs]: - """ - Method returns vulnerabilities for a list of certificate digests - - :param List[str] dgst_list: list of certificate digests - :return Dict[str, TransitiveCVE]: Dictionary of TransitiveCVE objects for specified certificate digests - """ - if not self._fitted: - raise ValueError("Finder not yet fitted") - - cert_vulnerabilities = {} - - for dgst in dgst_list: - cert_vulnerabilities[dgst] = self.predict_single_cert(dgst) - - return cert_vulnerabilities diff --git a/sec_certs/rules.yaml b/sec_certs/rules.yaml deleted file mode 100644 index 87589d72..00000000 --- a/sec_certs/rules.yaml +++ /dev/null @@ -1,1186 +0,0 @@ ---- - -##### -# Common Criteria certificate IDs, grouped by scheme (Alpha-2 ISO country code). -##### -cc_cert_id: - DE: - - "BSI-DSZ-CC-[0-9]+?-[0-9]+" - - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+-[0-9]+" - - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+" - - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(-[0-9][0-9][0-9][0-9])*" # German BSI (number + version + year or without year) - - "BSI-DSZ-CC-[0-9]+-[0-9][0-9][0-9][0-9]" # German BSI (number + year, no version) - - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(?!-)" # German BSI (number + version but no year => no - after version) - # - "BSI-DSZ-CC-[0-9]+" # Maybe? - FR: - - "ANSS[Ii](?:-|-CC-|-CC )[0-9]{4}/[0-9]+(v[1-9])?" # French - - "ANSS[Ii]-CC[ -][0-9]{4}[/-_][0-9][0-9]+(?!-M|-S|-R)" # French (/two or more digits then NOT -M or -S) - - "ANSS[Ii]-CC[ -][0-9]{4}[/-_][0-9]+(?:v[0-9])?[_/-][MSR][0-9]+" # French, maintenance or surveillance report (ANSSI-CC-2014_46_M01) - # 'ANSSI-CC-CER-F-.+?', # French - - "DCSS[Ii]-[0-9]+/[0-9]+" # French (DCSSI-2009/07) - - "Certification Report [0-9]+/[0-9]+" # French or Australia! Solved because we limit ourselves to scheme when doing heuristics. - - "Rapport de certification [0-9]+/[0-9]+" # French - NL: - - "NSCIB-CC-[0-9][0-9][0-9][0-9].+?" # Netherlands - - "NSCIB-CC-[0-9][0-9][0-9][0-9][0-9]*-CR" # Netherlands - - "NSCIB-CC-[0-9][0-9]-[0-9]+?-CR[0-9]+?" # Netherlands - - "NSCIB-CC-[0-9][0-9]-[0-9]+(-CR[0-9]+)*" # Netherlands (old number NSCIB-CC-05-6609 or NSCIB-CC-05-6609-CR) - - "NSCIB-CC-[0-9]+-CR[0-9]*" # Netherlands (new number NSCIB-CC-111441-CR NSCIB-CC-111441-CR1) - - "NSCIB-CC-[0-9]+-MA[0-9]*" # Netherlands (new number NSCIB-CC-222073-MA NSCIB-CC-200716-MA2) - - "NSCIB-CC-[0-9][0-9]-[0-9]+" # Netherlands (old number NSCIB-CC-05-6609) - - "NSCIB-CC-[0-9][0-9]-[0-9]+-CR[0-9]+" # Netherlands (NSCIB-CC-year2digits-number-CR) - "NO": - - "SERTIT-[0-9]+" # Norway - US: - - "CCEVS-VR-(?:CC-|VID)?[0-9]+-[0-9]+[a-z]?(?:-[0-9]+)?" # US NSA (CCEVS-VR-10884-2018 CCEVS-VR-VID10877-2018) - CA: - # '[0-9][0-9\-]+?-CR', # Canada - - "[0-9][0-9][0-9]-[347]-[0-9][0-9][0-9]?(?:-CR|P)?" # Canada xxx-{347}-xxx (383-4-438, 383-4-82-CR, 383-4-422P) - - "[0-9][0-9][0-9][ -](?:EWA|LSS|CCS)(?:[ -]20[0-9][0-9])?" # Canada (522-EWA-2020, 524 LSS 2020, 503-LSS) - - "[0-9][0-9][0-9](?:%20|-)(?:EWA|LSS|CCS)(?:%20|-)(?:20[0-9][0-9]%20|)CR%20v[0-9]\\.[0-9]" # Canada filename with space (518-LSS%20CR%20v1.0) - UK: - - "CRP[0-9]+[A-Z]?" # UK CESG - - "CERTIFICATION REPORT No. P[0-9]+[A-Z]?" # UK CESG - ES: - - "20[0-9][0-9][-‐][0-9]+[-‐]INF[-‐][0-9]+([-‐]?[ -‐](?:V|v)[0-9]+)?" # Spain ("2006-4-INF-98 v2" or "2006-4-INF-98-v2" or "2020-34-INF-3784- v1") - KR: - # Korea - # XXX: Do not use KECS-CR as those refer to the certificate report and do not represent the certificate id. - # - "KECS[-‐]CR[-‐][0-9]+[-‐][0-9]+" # Korea KECS-CR-20-61 - - "KECS[-‐](?:ISIS|NISS|CISS)[-‐][0-9]+[-‐][0-9]{4}" # Korea KECS-ISIS-1234-2011 - JP: - - "(?:CRP|ACR)-C[0-9]+-[0-9]+" # Japan (CRP-C0595-01 ACR-C0417-03) - - "JISEC-CC-CRP-C[0-9]+-[0-9]+-[0-9]+" # Japan (JISEC-CC-CRP-C0689-01-2020) - - "Certification No. [cC][0-9]+" # Japan (Certification No. C0090) - MY: - - "ISCB-[0-9]+-(?:RPT|FRM)-[CM][0-9]+[A-Z]?-(?:CR|AMR)(?:-[0-9])?-[vV][0-9](?:\\.[0-9])?[a-z]?" # Malaysia (ISCB-3-RPT-C092-CR-v1, ISCB-3-RPT-C068-CR-1-v1) - IT: - - "OCSI/CERT/.+?" # Italy - - "OCSI/CERT/.+?/20[0-9]+(?:\\w|/RC)" # Italy (OCSI/CERT/ATS/01/2018/RC) - TR: - - "[0-9\\.]+?/TSE-CCCS-[0-9]+" # Turkish CCCS (21.0.0sc/TSE-CCCS-75) - - "(?:[0-9]{1,2}\\.){2}[0-9]{1,2}/[0-9]{1,4}-[0-9]{3}" # 21.0.01/13-028 - SE: - - "CSEC ?[0-9]{6,7}" # Sweden (CSEC2019015) - IN: - # India (IC3S/DEL01/VALIANT/EAL1/0317/0007/CR STQC/CC/14-15/12/ETR/0017 IC3S/MUM01/CISCO/cPP/0119/0016/CR) - # will miss STQC/CC/14-15/12/ETR/0017 - - "(?:IC3S|STQC/CC)/[^ ]+? ?/CR" - SG: - - "CSA_CC_[0-9]+" # Singapore (CSA_CC_19001) - AU: - # Australia (EFS-T048 ETR 1.0, EFS-T056-ETR 1.0, DXC-EFC-T092-ETR 1.0) - # XXX: Do not use Australian ETR numbers, they are not certificate id. - # - "(?:EFS|EFT|DXC-EFC)-T[0-9]+(?: |-)ETR [0-9]+.[0-9]+" - - "Certificate Number: [0-9]{1,4}/[0-9]{1,4}" - - "Certification Report [0-9]+/[0-9]+" - -##### -# Common Criteria protection profile IDs, grouped by certification body (e.g. BSI) -##### -cc_protection_profile_id: - BSI: - - "BSI-(?:CC[-_]|)PP[-_]*.+?" - - "BSI-CCPP-.+?" - ANSSI: - - "ANSSI-CC-PP.+?" - KECS: - - "KECS-PP-[0-9]+-[0-9]+" - other: - - "PP-SSCD.+?" - - "PP_DBMS_.+?" - - "WBIS_V[0-9]\\.[0-9]" - - "EHCT_V.+?" - -##### -# Common Criteria security level (EAL or ITSEC). -##### -cc_security_level: - EAL: - - "EAL[ ]*[0-9+]+?" - - "EAL[ ]*[0-9] augmented" - ITSEC: - - "ITSEC[ ]*E[1-9]*.+?" - -##### -# Common Criteria security assurance requirement (SAR) code, grouped by class (e.g. ACE, ACM, ...). -##### -cc_sar: - ACE: - - "ACE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - ACM: - - "ACM(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - ACO: - - "ACO(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - ADO: - - "ADO(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - ADV: - - "ADV(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - AGD: - - "AGD(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - ALC: - - "ALC(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - ATE: - - "ATE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - AVA: - - "AVA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - AMA: - - "AMA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - APE: - - "APE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - ASE: - - "ASE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - -##### -# Common Criteria security functional requirement (SFR) code, grouped by class (e.g. FAU, FCO, ...). -##### -cc_sfr: - FAU: - - "FAU(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FCO: - - "FCO(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FCS: - - "FCS(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FDP: - - "FDP(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FIA: - - "FIA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FMT: - - "FMT(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FPR: - - "FPR(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FPT: - - "FPT(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FRU: - - "FRU(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FTA: - - "FTA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - FTP: - - "FTP(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" - -##### -# Common Criteria claim code, grouped by class (e.g. D, T, ...). -##### -cc_claims: - D: - - "D\\.[\\._\\-A-Z]+?" - O: - - "O\\.[\\._\\-A-Z]+?" - T: - - "T\\.[\\._\\-A-Z]+?" - A: - - "A\\.[\\._\\-A-Z]+?" - R: - - "R\\.[\\._\\-A-Z]+?" - OT: - - "OT\\.[\\._\\-A-Z]+?" - OP: - - "OP\\.[\\._\\-A-Z]+?" - OE: - - "OE\\.[\\._\\-A-Z]+?" - SA: - - "SA\\.[\\._\\-A-Z]+?" - OSP: - - "OSP\\.[\\._\\-A-Z]+?" - -##### -# A generic vendor of a product, mostly has smartcard/secure hardware vendors or other large vendors of certified products. -##### -vendor: - NXP: - - "NXP( Semiconductors)?( N\\.V\\.)?" - Infineon: - - "Infineon( Technologies)?( AG)?" - Samsung: - - "Samsung" - STMicroelectronics: - - "STMicroelectronics|STM|STMicro( N\\.V\\.)?" - Feitian: - - "Feitian( Technologies)?( Co.)?" - Gemalto: - - "Gemalto( N\\.V\\.)?" - Gemplus: - - "Gemplus( International)?( SA)?" - Axalto: - - "Axalto" - Thales: - - "Thales( Group)?( SA)?" - Oberthur: - - "(Oberthur|OBERTHUR)( Technologies| Card Systems)?" - Idemia: - - "Idemia|IDEMIA" - Sagem: - - "Sagem|SAGEM" - Morpho: - - "Morpho( Systèmes)?" - GD: - - "G&D|G\\+D|Giesecke\\+Devrient|Giesecke & Devrient" - Philips: - - "(Koninklijke )?Philips( N\\.V\\.)?" - Qualcomm: - - "Qualcomm" - Broadcom: - - "Broadcom( Inc.)?" - Huawei: - - "Huawei( Technologies)?( Co.)?" - Microsoft: - - "Microsoft( Corporation)?" - Cisco: - - "Cisco( Systems)?(, Inc.)?" - -##### -# Common Criteria evaluation facility, mostly from https://www.commoncriteriaportal.org/labs/, grouped roughly by facility. -##### -eval_facility: - Serma: - - "Serma Technologies|SERMA|Serma Safety & Security" - Thales: - - "THALES - CEACI|THALES/CNES" - Riscure: - - "Riscure" - SGS: - - "SGS" - - "SGS Bright[sS]ight" # SGS acquired BrightSight - BrightSight: - - "Bright[sS]ight" - Applus: - - "Applus Laboratories" - TUV: - - "(tuvit|TÜViT|TUViT|TÜV Informationstechnik|TUV Informationstechnik)" - CESTI: - - "CESTI" - DXC: - - "DXC Technology" - Teron: - - "Teron Labs" - EWA: - - "(EWA|EWA-Canada)" - Lightship: - - "Lightship Security" - AMOSSYS: - - "AMOSSYS" - CEA-LETI: - - "(CEA - LETI|CEA/LETI|CEA-LETI)" - OPPIDA: - - "OPPIDA" - TrustedLabs: - - "Trusted Labs" - atsec: - - "atsec" - DeutscheTelekom: - - "Deutsche Telekom Security" - DFKI: - - "(Deutsches Forschungszentrum für künstliche Intelligenz|dfki|DFKI)" - MTG: - - "MTG AG" - secuvera: - - "secuvera" - SRC: - - "SRC Security Research & Consulting" - Acucert: - - "Acucert Labs" - ERTL: - - "Common Criteria Test Laboratory,? ERTL" - ETDC: - - "Common Criteria Test Laboratory,? ETDC" - CCLab: - - "CCLab Software Laboratory" - Deeplab: - - "Deeplab" - IMQLPS: - - "IMQ/LPS" - LVSLeonardo: - - "LVS Leonardo" - LVSTechnisBlu: - - "LVS Technis Blu" - ECSEC: - - "ECSEC Laboratory" - ITSC: - - "Information Technology Security Center" - Acumen: - - "Acumen Security" - BoozAllenHamilton: - - "Booz Allen Hamilton" - Gossamer: - - "Gossamer Security" - Leidos: - - "Leidos" - UL: - - "UL Verification Services" - BEAM: - - "BEAM Teknoloji" - Certby: - - "Certby Lab" - DEKRA: - - "DEKRA Testing and Certification" - STMITSEF: - - "STM ITSEF" - TUBITAK-BILGEM: - - "TÜBİTAK BİLGEM" - Combitech: - - "Combitech AB" - Intertek: - - "Intertek" - Clover: - - "Clover Technologies" - LAYAKK: - - "LAYAKK SEGURIDAD INFORMATICA" - An: - - "An Security" - TSystems: - - "T-Systems International" - KISA: - - "KISA" - KOIST: - - "KOIST" - KSEL: - - "KSEL" - KOSYAS: - - "KOSYAS" - KTR: - - "KTR" - KTC: - - "KTC" - TTA: - - "TTA" - ADS: - - "Advanced Data Security" - Nemko: - - "Nemko System Sikkerhet" - Norconsult: - - "Norconsult( AS)?" - Secura: - - "Secura" - BAE: - - "BAE Applied Intelligence" - -##### -# Symmetric crypto primitive (e.g. a block or stream cipher), grouped by competition/standardization effort. -##### -symmetric_crypto: - AES_competition: - AES: - - "AES-?(?P128|192|256)?" - Rijndael: - - "Rijndael" - Twofish: - - "Twofish" - Serpent: - - "Serpent" - MARS: - - "MARS" - HPC: - - "HPC" - FROG: - - "FROG" - CAST: - - "CAST-?(?P128|160|192|224|256|5)?" - RC: - - "RC(?P[2456])" - CRYPTON: - - "CRYPTON" - DEAL: - - "DEAL" - E2: - - "E2" - LOKI97: - - "LOKI97" - MAGENTA: - - "MAGENTA" - SAFER: - - "SAFER\\+" - DES: - DES: - - "DE[SA]" - 3DES: - - "([3T]|Triple)-?DE[SA]" - Lucifer: - - "Lucifer" - djb: - ChaCha: - - "[XH]?ChaCha(?P20|8)?" - Salsa: - - "[XH]?Salsa(?P20(/8|/12)?)?" - Poly: - - "Poly1305" - LWC_competition: - ASCON: - - "(ASCON|Ascon)" - Elephant: - - "Elephant" - GIFT: - - "GIFT(-COFB)?" - Grain: - - "Grain128(-AEAD)?" - ISAP: - - "ISAP" - PhotonBeetle: - - "Photon-?Beetle" - Romulus: - - "Romulus" - Sparkle: - - "Sparkle" - TinyJambu: - - "TinyJambu" - Xoodyak: - - "Xoodyak" - Gimli: - - "Gimli" - eSTREAM: - HC: - - "HC-[0-9]{3}" - Rabbit: - - "Rabbit" - SOSEMANUK: - - "SOSEMANUK" - MICKEY: - - "MICKEY(-128)?" - Trivium: - - "Trivium" - CAESAR: - ACORN: - - "ACORN" - AEGIS: - - "AEGIS(-128)?" - Deoxys: - - "Deoxys(-2)?" - COLM: - - "COLM" - miscellaneous: - IDEA: - - "IDEA" - Blowfish: - - "Blowfish" - Camellia: - - "Camellia" - ARIA: - - "ARIA" - SM4: - - "SM4" - GOST: - - "GOST 28147-89" - - "Magma" - - "Kuznyechik" - SEED: - - "SEED" - Skipjack: - - "Skipjack" - Skinny: - - "Skinny|SKINNY" - constructions: - MAC: - - "HMAC" - - "HMAC-[A-Z]+?-(?:160|224|256|384|512)" - - "KMAC" - - "CMAC|CBC-MAC" - -##### -# Asymmetric crypto primitive, grouped by type (RSA, ECC, FF). -##### -asymmetric_crypto: - RSA: - - "RSA[- ]?(?P512|768|1024|1280|1536|2048|3072|4096|8192)" - - "RSASSAPKCS1-[Vv]1_5" - - "RSA-?(OAEP|PSS|CRT)" - ECC: - ECDH: - - "ECDHE?" - ECDSA: - - "ECDSA" - EdDSA: - - "EdDSA" - ECIES: - - "ECIES" - ECC: - - "ECC" - FF: - DH: - - "Diffie-Hellman|DHE?" - DSA: - - "DSA" - -##### -# Post-quantum crypto primitive, grouped by primitive, from NIST-PQC. -##### -pq_crypto: - ClassicMcEliece: - - "Classic[ -]McEliece" - Kyber: - - "(CRYSTALS-)?(Kyber|KYBER)" - NTRU: - - "NTRU" - NTRUPrime: - - "NTRU[ -]?Prime" - Saber: - - "Saber|SABER" - Dilithium: - - "(CRYSTALS-)?(Dilithium|DILITHIUM)" - Falcon: - - "Falcon|FALCON" - Rainbow: - - "Rainbow" - UOV: - - "UOV" - BIKE: - - "BIKE" - Frodo: - - "Frodo(KEM)?" - HQC: - - "HQC" - SIKE: - - "SIKE" - SIDH: - - "SIDH" - GeMSS: - - "GeMSS" - Picnic: - - "Picnic" - SPHINCS: - - "SPHINCS\\+" - -##### -# Hash-function, grouped by hash-function class (SHA, MD) or competition (PHC). -##### -hash_function: - SHA: - SHA1: - - "SHA-?1" - SHA2: - - "SHA-?(?P160|224|256|384|512)" - - "SHA-?2" - SHA3: - - "SHA-?3(-[0-9]{3})?" - Keccak: - - "Keccak" - SHAKE: - - "SHAKE[0-9]{3}" - Groestl: - - "(Groestl|Grøstl)" - BLAKE: - - "(Blake|BLAKE)[23][sbX]?" - JH: - - "JH" - Skein: - - "Skein" - MD: - MD4: - - "MD4" - MD5: - - "MD5" - MD6: - - "MD6" - RIPEMD: - - "RIPEMD(-?[0-9]{3})?" - Streebog: - - "Streebog" - Whirpool: - - "Whirpool" - PHC_competition: - Argon: - - "Argon2?[id]?" - battcrypt: - - "battcrypt" - Catena: - - "Catena" - Lyra2: - - "Lyra2" - Makwa: - - "Makwa" - POMELO: - - "POMELO" - Pufferfish: - - "Pufferfish" - yescrypt: - - "yescrypt" - bcrypt: - - "bcrypt" - scrypt: - - "scrypt" - PBKDF: - - "PBKDF[12]?" - -##### -# General cryptographic scheme. -##### -crypto_scheme: - MAC: - - "MAC" - KEM: - - "KEM" - PKE: - - "PKE" - KEX: - - "KEX|Key [eE]xchange" - KA: - - "KA|Key [aA]greement" - PAKE: - - "PAKE" - AEAD: - - "AEAD" - -##### -# General cryptographic protocol. -##### -crypto_protocol: - SSH: - - "SSH" - TLS: - SSL: - - "SSL( ?v?[123]\\.0)?" - TLS: - - "TLS( ?v?1\\.[0123])?" - DTLS: - - "DTLS( ?v?1\\.[0123])?" - PACE: - - "PACE" - IKE: - - "IKE(v[12])?" - IPsec: - - "IPsec" - VPN: - - "VPN" - PGP: - - "PGP" - -##### -# Random number generator. -##### -randomness: - DUAL_EC: - - "DUAL_EC_DRBG" - TRNG: - - "D?TRNG" - PRNG: - - "PRNG" - - "DRBG" - RNG: - - "RN[GD]" - - "RBG" - -##### -# Block cipher mode. -##### -cipher_mode: - ECB: - - "ECB" - CBC: - - "CBC" - CTR: - - "CTR" - CFB: - - "CFB" - OFB: - - "OFB" - GCM: - - "GCM" - SIV: - - "SIV" - XTR: - - "XTR" - CCM: - - "CCM" - LRW: - - "LRW" - XEX: - - "XEX" - XTS: - - "XTS" - -##### -# An elliptic curve, grouped by standardization body (e.g. NIST, Brainpool). -# Note that multiple curve names may correspond to the same curve. -##### -ecc_curve: - NIST: - - "(?:Curve |curve |)P-(192|224|256|384|521)" - - "(?:ansit|ansip|ANSIP|ANSIT)[0-9]+?[rk][12]" - - "(NIST)? ?[PBK]-[0-9]{3}" - - "(?:secp|sect|SECP|SECT)[0-9]+?[rk][12]" - - "prime[0-9]{3}v[123]" - - "c2[pto]nb[0-9]{3}[vw][123]" - Brainpool: - - "(?:brainpool|BRAINPOOL)P[0-9]{3}[rkt][12]" - ANSSI: - - "(?:anssi|ANSSI)?[ ]*FRP[0-9]+?v1" - NUMS: - - "numsp[0-9]{3}[td]1" - Curve: - - "Curve(25519|1174|4417|22103|67254|383187|41417)" - Edwards: - - "Ed(25519|448)" - ssc: - - "ssc-(160|192|224|256|288|320|384|512)" - Tweedle: - - "Tweedle(dee|dum)" - Pasta: - - "(Pallas|Vesta)" - JubJub: - - "JubJub" - BLS: - - "BLS(12|24)-[0-9]{3}" - BN: - - "bn[0-9]{3}" - -##### -# A cryptographic engine. -##### -crypto_engine: - TORNADO: - - "TORNADO" - SmartMX: - - "SmartMX2?" - NexCrypt: - - "NexCrypt" - -##### -# TLS cipher suite name. -##### -tls_cipher_suite: - TLS: - - "TLS(_[A-Z0-9]+){1,3}_WITH(_[A-Z0-9]+){2,4}" - -##### -# A cryptographic library, grouped by rough library category. -##### -crypto_library: - Neslib: - - "(?:NesLib|NESLIB) [v]*[0-9\\.]+" - AT1: - - "AT1 Secure .{1,30}? Library [v]*[0-9\\.]+" - - "AT1 Secure RSA/ECC/SHA library" - Generic: - - "Crypto Library [v]*[0-9\\.]+" - AtmelToolbox: - - "(ATMEL|Atmel) Toolbox [0-9\\.]+" - Infineon: - - "v1\\.02\\.013" # Infineon's ROCA-vulnerable library - OpenSSL: - - "OpenSSL" - LibreSSL: - - "LibreSSL" - BoringSSL: - - "BoringSSL" - MatrixSSL: - - "MatrixSSL" - Nettle: - - "Nettle" - GnuTLS: - - "GnuTLS" - libtomcrypt: - - "libtomcrypt" - BearSSL: - - "BearSSL" - Botan: - - "Botan" - Crypto++: - - "Crypto\\+\\+" - wolfSSL: - - "wolfSSL" - mbedTLS: - - "[mM]bedTLS" - s2n: - - "(Amazon )?s2n" - NSS: - - "NSS" - libgcrypt: - - "libgcrypt" - BouncyCastle: - - "BouncyCastle" - cryptlib: - - "cryptlib" - NaCl: - - "NaCl" - libsodium: - - "libsodium" - libsecp256k1: - - "libsecp256k1" - -##### -# A vulnerability idenfitier or name (e.g. CVE-... but also Minerva, ROCA). -##### -vulnerability: - CVE: - - "CVE-[0-9]+?-[0-9]+?" - CWE: - - "CWE-[0-9]+?" - ROCA: - - "ROCA" - Minerva: - - "Minerva" - TPM-Fail: - - "TPM[\\.-]Fail" - -##### -# A side-channel analysis related term, grouped into SCA, FI and other. -##### -side_channel_analysis: - SCA: - - "Leak-Inherent" - - "[Pp]hysical [Pp]robing" - - "[Ss]ide.channels?" - - "SPA|DPA" - - "[tT]iming [aA]ttacks?" - - "[Tt]emplate [aA]ttacks?" - - "[Pp]rofiled [aA]ttacks?" - - "[Cc]lustering [aA]ttacks?" - - "t-test|TVLA" - FI: - - "[pP]hysical [tT]ampering" - - "[Mm]alfunction" - - "DFA|SIFA" - - "[Ff]+ault [iI]nduction" - - "[Ff]+ault [iI]njection" - other: - - "[Dd]eep[ -][lL]earning" - - "[Cc]old [bB]oot" - - "[Rr]ow-?hammer" - - "[Rr]everse [eE]ngineering" - - "[Ll]attice [aA]ttacks?" - - "[Oo]racle [aA]ttacks?" - - "[Bb]leichenbacher [aA]ttacks?" - - "[Bb]ellcore [aA]ttacks?" - - "JIL(-(AAPS|COMP|AM|AAPHD|AMHD))?" - - "JHAS" - -##### -# A term used in the certification process. -##### -certification_process: - OutOfScope: - - "[oO]ut of [sS]cope" - - "[\\.\\(].{0,100}?.[oO]ut of [sS]cope..{0,100}?[\\.\\)]" - - ".{0,100}[oO]ut of [sS]cope.{0,100}" - ConfidentialDocument: - - ".{0,100}confidential document.{0,100}" - SecurityFunction: - - "[sS]ecurity [fF]unction SF\\.[a-zA-Z0-9_]" - -##### -# A technical report id, grouped by standardization body (e.g. BSI). -##### -technical_report_id: - BSI: - - "BSI[ ]*TR-[0-9]+?(?:-[0-9]+?|)" - - "BSI [0-9]+?" # German BSI document containing list of issued certificates in some period - -##### -# A device model, grouped by manufacturer and subgroups into particular model families. -##### -device_model: - G87: - - "G87-.+?" - ATMEL: - - "ATMEL AT.+?" - STM: - STM32: - - "STM32[FGLHW][0-7][0-9]{1,2}[FGKTSCRVZI][468BCDEFGHI][PHUTY][67]" - Infineon: - SLE: - - "SLE[0-9]{2}[A-Z]{3}[0-9]{1-4}[A-Z]{1-3}" - -##### -# A Trusted Execution Environment, grouped by manufacturer (e.g. Intel, ARM, ...). -##### -tee_name: - Intel: - - "(Intel )?SGX" - ARM: - - "(ARM )?TrustZone" - - "(ARM )?(Realm Management Extension|Confidential Compute Architecture)" - AMD: - - "(AMD )?(PSP|Platform Security Processor)" - - "(AMD )?(SEV|Secure Encrypted Virtualization)" - IBM: - - "(IBM )?(SSC|Secure Service Container)" - - "(IBM )?(SE|Secure Execution)" - other: - - "Cloud Link TEE" - - "iOS Secure Enclave" - - "iTrustee" - - "Trusty" - - "OPTEE" - - "QTEE" - - "TEEgris" - - "T6" - - "Kinibi" - - "SW TEE" - - "WatchTrust" - - "TEE" - -##### -# An OS name, grouped by OS. -##### -os_name: - STARCOS: - - "STARCOS(?: [0-9\\.]+?|)" - JCOP: - - "JCOP[ ]*[0-9]" - -##### -# CPLC data name.. -##### -cplc_data: - ICFab: - - "IC[ \\.]*Fabricator" - ICType: - - "IC[ \\.]*Type" - ICVersion: - - "IC[ \\.]*Version" - -##### -# An elementary data group. -##### -ic_data_group: - EF: - - "EF\\.DG[1-9][0-6]?" - - "EF\\.COM" - - "EF\\.CardAccess" - - "EF\\.SOD" - - "EF\\.ChipSecurity" - -##### -# Standard ID, grouped by standardization body (e.g. FIPS, NIST, ISO). -##### -standard_id: - FIPS: - - "FIPS ?(?:PUB )?[0-9]+(-[0-9]+)?" - NIST: - - "(NIST )?SP [0-9]+-[0-9]+?[a-zA-Z]?" - PKCS: - - "PKCS[ #]*[1-9]+" - BSI: - - "BSI-AIS[ ]*[0-9]+?" - - "AIS[ ]*[0-9]+?" - RFC: - - "RFC[ -]?[0-9]+?" - ISO: - - "ISO/IEC[ ]*[0-9]+[-]*[0-9]*" - - "ISO/IEC[ ]*[0-9]+:[ 0-9]+" - - "ISO/IEC[ ]*[0-9]+" - ICAO: - - "ICAO(?:-SAC|)" - X509: - - "[Xx]\\.509" - SCP: - - "(?:SCP|scp)[ ']*[0-9][0-9]" - CC: - - "CC[I]*MB-20[0-9]+?-[0-9]+?-[0-9]+?" # Common Criteria methodology - - "CCIMB-9[0-9]-[0-9]+?" # Common Criteria methodology old - -##### -# JavaCard version identifier. -##### -javacard_version: - JavaCard: - - "(?:Java Card|JavaCard) [2-3]\\.[0-9](?:\\.[0-9]|)" - - "JC[2-3]\\.[0-9](?:\\.[0-9]|)" - - "(?:Java Card|JavaCard) \\(version [2-3]\\.[0-9](?:\\.[0-9]|)\\)" - GlobalPlatform: - - "(?:Global Platform|GlobalPlatform) [2-3]\\.[0-9]\\.[0-9]" - - "(?:Global Platform|GlobalPlatform) \\(version [2-3]\\.[0-9]\\.[0-9]\\)" - -##### -# JavaCard API constant, grouped into "ALG", "misc" and "curves". -##### -javacard_api_const: - ALG: - RNG: - - "ALG_(?:PSEUDO_RANDOM|SECURE_RANDOM|TRNG|PRESEEDED_DRBG|FAST|KEYGENERATION)" - DES: - - "ALG_DES_[A-Z_0-9]+" - RSA: - - "ALG_RSA_[A-Z_0-9]+" - DSA: - - "ALG_DSA_[A-Z_0-9]+" - ECDSA: - - "ALG_ECDSA_[A-Z_0-9]+" - AES: - - "ALG_AES_[A-Z_0-9]+" - HMAC: - - "ALG_HMAC_[A-Z_0-9]+" - KOREAN: - - "ALG_KOREAN_[A-Z_0-9]+" - EC: - - "ALG_EC_[A-Z_0-9]+?" - SHA: - - "ALG_SHA_[A-Z_0-9]+" - SHA3: - - "ALG_SHA3_[A-Z_0-9]+" - MD: - - "ALG_MD[A-Z_0-9]+" - RIPEMD: - - "ALG_RIPEMD[A-Z_0-9]+" - ISO3309: - - "ALG_ISO3309_[A-Z_0-9]+" - XDH: - - "ALG_XDH" - SM2: - - "ALG_SM2" - SM3: - - "ALG_SM3" - NULL: - - "ALG_NULL" - misc: - - "SIG_CIPHER_[A-Z_0-9]+" - - "CIPHER_[A-Z_0-9]+" - - "PAD_[A-Z_0-9]+" - - "TYPE_[A-Z_0-9]+" - - "LENGTH_[A-Z_0-9]+" - - "OWNER_PIN[A-Z_0-9]*" - curves: - - "BRAINPOOLP[A-Z_0-9]+(?:R|T)1" - - "ED25519" - - "ED448" - - "FRP256V1" - - "SECP[0-9]*R1" - - "SM2" - - "X25519" - - "X448" - -##### -# JavaCard common package identifiers. -##### -javacard_packages: - java: - - "java\\.[a-z\\.]+" - javacard: - - "javacard\\.[a-z\\.]+" - javacardx: - - "javacardx\\.[a-z\\.]+" - org: - - "org\\.[0-9a-z\\.]+" - uicc: - - "uicc\\.[a-z\\.]+" - com: - - "com\\.[0-9a-z\\.]+" - de: - - "de\\.bsi\\.[a-z\\.]+" - -##### -# FIPS 140 certificate id. -##### -fips_cert_id: - Cert: - - "(?:#[^\\S\\r\\n]?|Cert\\.?(?!.\\s)[^\\S\\r\\n]?|Certificate[^\\S\\r\\n]?)(?P\\d{1,4})(?!\\d)" - -##### -# FIPS 140 security level. -##### -fips_security_level: - Level: - - "[lL]evel (\\d)" - -##### -# FIPS 140 "certlike" string, that needs to get removed from certificate id matches. -##### -fips_certlike: - Certlike: - # --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) --- - # + added (and #id) everywhere - - "HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\\(?(?: |hardware|firmware)*?[\\s(\\[]*?(?:#|cert\\.?|Cert\\.?|Certificate|sample)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:[\\s#]*and[\\s#]*\\d+)?" - # --- same as above, without hw or fw --- - - "HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" - # --- SHS/A - (bits) (method) ((cert #) numbers) --- - - "SH[SA][-– 123]*(?:;|\\/|160|224|256|384|512)?(?:[\\s(\\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\\s,]*?[\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:\\)?\\[#?\\d+\\])?(?:[\\s#]*?and[\\s#]*?\\d+)?" - # --- RSA (bits) (method) ((cert #)) --- - - "RSA(?:[-– ]*(?:;|\\/|512|768|1024|1280|1536|2048|3072|4096|8192)\\s\\(\\[]*?(?:(?:;|\\/|KAT|Verify|PSS|\\s)*?)?[\\s,]*?[\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" - # --- RSA (SSA) (PKCS) (version) (#) --- - - "(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\\s?#?\\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\\s#]*?(\\d{1,4})?" - # --- AES (bits) (method) ((cert #)) --- - - "AES[-– ]*((?: |;|\\/|bit|key|128|192|256|CBC)*(?: |\\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:\\)?[\\s#]*?\\[#?\\d+\\])?(?:[\\s#]*?and[\\s#]*?(\\d+))?" - # --- Diffie Helman (CVL) ((cert #)) --- - - "Diffie[-– ]*Hellman[,\\s(\\[]*?(?:CVL|\\s)*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?[\\s#]*?(\\d{1,4})" - # --- DRBG (bits) (method) (cert #) --- - - "DRBG[ –-]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" - # --- DES (bits) (method) (cert #) - - "DES[ –-]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\\d(?: and \\d)? keying options?))*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)*?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:[\\s#]*?and[\\s#]*?(\\d+))?" - # --- DSA (bits) (method) (cert #) - - "DSA[ –-]*((?:;|\\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" - # --- platforms (#)+ - this is used in modification history --- - - "[Pp]latforms? #\\d+(?:#\\d+|,| |-|and)*[^\\n]*" - # --- CVL (#) --- - - "CVL[\\s#]*?(\\d{1,4})" - # --- PAA (#) --- - - "PAA[: #]*?\\d{1,4}" - # --- (#) Type --- - - "(?:#|cert\\.?|sample|Cert\\.?|Certificate)[\\s#]*?(\\d+)?\\s*?(?:AES|SHS|SHA|RSA|HMAC|Diffie-Hellman|DRBG|DES|CVL)" - # --- PKCS (#) --- - - "PKCS[\\s]?#?\\d+" - - "PKSC[\\s]?#?\\d+" # typo, #625 - # --- # C and # A (just in case) --- - - "#\\s+?[Cc]\\d+" - - "#\\s+?[Aa]\\d+" - - -##### -# Common Criteria rules. -##### -cc_rules: - - "cc_cert_id" - - "cc_protection_profile_id" - - "cc_security_level" - - "cc_sar" - - "cc_sfr" - - "cc_claims" - - "vendor" - - "eval_facility" - - "symmetric_crypto" - - "asymmetric_crypto" - - "pq_crypto" - - "hash_function" - - "crypto_scheme" - - "crypto_protocol" - - "randomness" - - "cipher_mode" - - "ecc_curve" - - "crypto_engine" - - "tls_cipher_suite" - - "crypto_library" - - "vulnerability" - - "side_channel_analysis" - - "technical_report_id" - - "device_model" - - "tee_name" - - "os_name" - - "cplc_data" - - "ic_data_group" - - "standard_id" - - "javacard_version" - - "javacard_api_const" - - "javacard_packages" - - "certification_process" - - -##### -# FIPS rules. -##### -fips_rules: - - "fips_cert_id" - - "fips_security_level" - - "fips_certlike" - - "vendor" - - "eval_facility" - - "symmetric_crypto" - - "asymmetric_crypto" - - "pq_crypto" - - "hash_function" - - "crypto_scheme" - - "crypto_protocol" - - "randomness" - - "cipher_mode" - - "ecc_curve" - - "crypto_engine" - - "tls_cipher_suite" - - "crypto_library" - - "vulnerability" - - "side_channel_analysis" - - "device_model" - - "tee_name" - - "os_name" - - "cplc_data" - - "ic_data_group" - - "standard_id" - - "javacard_version" - - "javacard_api_const" - - "javacard_packages" - - "certification_process" diff --git a/sec_certs/sample/__init__.py b/sec_certs/sample/__init__.py deleted file mode 100644 index ecbbd541..00000000 --- a/sec_certs/sample/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -"""This package holds mostly data objects of primary interest (Common Criteria, FIPS), or assisting objects -like CPE, CVE, etc. The objects mostly hold data and allow for serialization, but can also perform some basic transformations. -""" - -from sec_certs.sample.cc_certificate_id import CertificateId -from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.cpe import CPE, cached_cpe -from sec_certs.sample.cve import CVE -from sec_certs.sample.fips import FIPSCertificate -from sec_certs.sample.fips_algorithm import FIPSAlgorithm -from sec_certs.sample.fips_iut import IUTEntry, IUTSnapshot -from sec_certs.sample.fips_mip import MIPEntry, MIPSnapshot, MIPStatus -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.sample.sar import SAR - -__all__ = [ - "CertificateId", - "CommonCriteriaMaintenanceUpdate", - "CommonCriteriaCert", - "CPE", - "cached_cpe", - "CVE", - "FIPSCertificate", - "FIPSAlgorithm", - "IUTEntry", - "IUTSnapshot", - "MIPEntry", - "MIPSnapshot", - "MIPStatus", - "ProtectionProfile", - "SAR", -] diff --git a/sec_certs/sample/cc_certificate_id.py b/sec_certs/sample/cc_certificate_id.py deleted file mode 100644 index 428ca73b..00000000 --- a/sec_certs/sample/cc_certificate_id.py +++ /dev/null @@ -1,151 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass - - -@dataclass(eq=True, frozen=True) -class CertificateId: - """ - A Common Criteria certificate id. - """ - - scheme: str - raw: str - - def _canonical_fr(self) -> str: - new_cert_id = self.clean - rules = [ - "(?:Rapport de certification|Certification Report) ([0-9]+[/-_][0-9]+(?:v[1-9])?(?:[_/-][MSR][0-9]+)?)", - "(?:ANSS[Ii]|DCSSI)(?:-CC)?[- ]([0-9]+[/-_][0-9]+(?:v[1-9])?(?:[_/-][MSR][0-9]+)?)", - "([0-9]+[/-_][0-9]+(?:v[1-9])?(?:[_/-][MSR][0-9]+)?)", - ] - for rule in rules: - if match := re.match(rule, new_cert_id): - return "ANSSI-CC-" + match.group(1).replace("_", "/") - - return new_cert_id - - def _canonical_de(self) -> str: - def extract_parts(bsi_parts: list[str]) -> tuple: - cert_num = None - cert_version = None - cert_year = None - - if len(bsi_parts) > 3: - cert_num = bsi_parts[3] - if len(bsi_parts) > 4: - if bsi_parts[4].startswith("V") or bsi_parts[4].startswith("v"): - cert_version = bsi_parts[4].upper() # get version in uppercase - else: - cert_year = bsi_parts[4] - if len(bsi_parts) > 5: - cert_year = bsi_parts[5] - - return cert_num, cert_version, cert_year - - bsi_parts = self.clean.split("-") - - cert_num, cert_version, cert_year = extract_parts(bsi_parts) - - # reconstruct BSI number again - new_cert_id = "BSI-DSZ-CC" - if cert_num is not None: - new_cert_id += "-" + cert_num - if cert_version is not None: - new_cert_id += "-" + cert_version - if cert_year is not None: - new_cert_id += "-" + cert_year - - return new_cert_id - - def _canonical_es(self) -> str: - cert_id = self.clean - spain_parts = cert_id.split("-") - cert_year = spain_parts[0] - cert_batch = spain_parts[1].lstrip("0") - cert_num = spain_parts[3].lstrip("0") - - if "v" in cert_num: - cert_num = cert_num[: cert_num.find("v")] - if "V" in cert_num: - cert_num = cert_num[: cert_num.find("V")] - - new_cert_id = f"{cert_year}-{cert_batch}-INF-{cert_num.strip()}" # drop version # TODO: Maybe do not drop? - - return new_cert_id - - def _canonical_it(self): - new_cert_id = self.clean - if not new_cert_id.endswith("/RC"): - new_cert_id = new_cert_id + "/RC" - - return new_cert_id - - def _canonical_in(self): - return self.clean.replace(" ", "") - - def _canonical_se(self): - return self.clean.replace(" ", "") - - def _canonical_uk(self): - new_cert_id = self.clean - if match := re.match("CERTIFICATION REPORT No. P([0-9]+[A-Z]?)", new_cert_id): - new_cert_id = "CRP" + match.group(1) - return new_cert_id - - def _canonical_ca(self): - new_cert_id = self.clean - if new_cert_id.endswith("-CR"): - new_cert_id = new_cert_id[:-3] - if new_cert_id.endswith("P"): - new_cert_id = new_cert_id[:-1] - return new_cert_id.replace(" ", "-") - - def _canonical_jp(self): - new_cert_id = self.clean - if match := re.match("Certification No. (C[0-9]+)", new_cert_id): - return match.group(1) - if match := re.search("CRP-(C[0-9]+)-", new_cert_id): - return match.group(1) - return new_cert_id - - def _canonical_no(self): - new_cert_id = self.clean - cert_num = int(new_cert_id.split("-")[1]) - return f"SERTIT-{cert_num:03}" - - @property - def clean(self) -> str: - """ - The clean version of this certificate id. - """ - return self.raw.replace("\N{HYPHEN}", "-").strip() - - @property - def canonical(self) -> str: - """ - The canonical version of this certificate id. - """ - # We have rules for some schemes to make canonical cert_ids. - schemes = { - "FR": self._canonical_fr, - "DE": self._canonical_de, - "ES": self._canonical_es, - "IT": self._canonical_it, - "IN": self._canonical_in, - "SE": self._canonical_se, - "UK": self._canonical_uk, - "CA": self._canonical_ca, - "JP": self._canonical_jp, - "NO": self._canonical_no, - } - - if self.scheme in schemes: - return schemes[self.scheme]() - else: - return self.clean - - -def canonicalize(cert_id_str: str, scheme: str) -> str: - return CertificateId(scheme, cert_id_str).canonical diff --git a/sec_certs/sample/cc_maintenance_update.py b/sec_certs/sample/cc_maintenance_update.py deleted file mode 100644 index abe3b176..00000000 --- a/sec_certs/sample/cc_maintenance_update.py +++ /dev/null @@ -1,99 +0,0 @@ -from __future__ import annotations - -import logging -from datetime import date -from typing import ClassVar - -import sec_certs.utils.helpers as helpers -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.serialization.json import ComplexSerializableType - -logger = logging.getLogger(__name__) - - -class CommonCriteriaMaintenanceUpdate(CommonCriteriaCert, ComplexSerializableType): - pandas_columns: ClassVar[list[str]] = [ - "dgst", - "name", - "report_link", - "st_link", - "related_cert_digest", - "maintenance_date", - ] - - def __init__( - self, - name: str, - report_link: str, - st_link: str, - state: CommonCriteriaCert.InternalState | None, - pdf_data: CommonCriteriaCert.PdfData | None, - heuristics: CommonCriteriaCert.Heuristics | None, - related_cert_digest: str, - maintenance_date: date, - ): - super().__init__( - "", - "", - name, - "", - "", - "", - None, - None, - report_link, - st_link, - "", - "", - set(), - set(), - state, - pdf_data, - heuristics, - ) - self.related_cert_digest = related_cert_digest - self.maintenance_date = maintenance_date - - @property - def serialized_attributes(self) -> list[str]: - return ["dgst"] + list(self.__class__.__init__.__code__.co_varnames)[1:] - - @property - def dgst(self) -> str: - if not self.name: - raise RuntimeError("MaintenanceUpdate digest can't be computed, because name of update is missing.") - return "cert_" + self.related_cert_digest + "_update_" + helpers.get_first_16_bytes_sha256(self.name) - - @property - def pandas_tuple(self) -> tuple: - return tuple([getattr(self, x) for x in CommonCriteriaMaintenanceUpdate.pandas_columns]) - - @classmethod - def from_dict(cls, dct: dict) -> CommonCriteriaMaintenanceUpdate: - dct.pop("dgst") - return cls(*(tuple(dct.values()))) - - @classmethod - def get_updates_from_cc_cert(cls, cert: CommonCriteriaCert) -> list[CommonCriteriaMaintenanceUpdate]: - if cert.maintenance_updates is None: - raise RuntimeError("Dataset was probably not built correctly - this should not be happening.") - - return [ - cls( - x.maintenance_title, - x.maintenance_report_link, - x.maintenance_st_link, - None, - None, - None, - cert.dgst, - x.maintenance_date, - ) - for x in cert.maintenance_updates - if ( - x.maintenance_title is not None - and x.maintenance_report_link is not None - and x.maintenance_st_link is not None - and x.maintenance_date is not None - ) - ] diff --git a/sec_certs/sample/certificate.py b/sec_certs/sample/certificate.py deleted file mode 100644 index bb49c0df..00000000 --- a/sec_certs/sample/certificate.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import annotations - -import copy -import logging -from abc import ABC, abstractmethod -from collections import ChainMap -from dataclasses import dataclass, field -from typing import Any, Generic, TypeVar - -import sec_certs.utils.extract -from sec_certs.cert_rules import PANDAS_KEYWORDS_CATEGORIES -from sec_certs.serialization.json import ComplexSerializableType - -logger = logging.getLogger(__name__) - -T = TypeVar("T", bound="Certificate") -H = TypeVar("H", bound="Heuristics") -P = TypeVar("P", bound="PdfData") - - -@dataclass -class References(ComplexSerializableType): - directly_referenced_by: set[str] | None = field(default=None) - indirectly_referenced_by: set[str] | None = field(default=None) - directly_referencing: set[str] | None = field(default=None) - indirectly_referencing: set[str] | None = field(default=None) - - -class Heuristics: - cpe_matches: set[str] | None - related_cves: set[str] | None - - -class PdfData: - def get_keywords_df_data(self, var: str) -> dict[str, float]: - data_dct = getattr(self, var) - return dict( - ChainMap( - *[ - sec_certs.utils.extract.get_sums_for_rules_subset(data_dct, cat) - for cat in PANDAS_KEYWORDS_CATEGORIES - ] - ) - ) - - -class Certificate(Generic[T, H, P], ABC, ComplexSerializableType): - manufacturer: str | None - name: str | None - pdf_data: P - heuristics: H - - def __init__(self, *args, **kwargs): - pass - - def __repr__(self) -> str: - return str(self.to_dict()) - - def __str__(self) -> str: - return "Not implemented" - - @property - @abstractmethod - def dgst(self): - raise NotImplementedError("Not meant to be implemented") - - @property - @abstractmethod - def label_studio_title(self): - raise NotImplementedError("Not meant to be implemented") - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Certificate): - return False - return self.dgst == other.dgst - - def to_dict(self) -> dict[str, Any]: - return { - **{"dgst": self.dgst}, - **{key: val for key, val in copy.deepcopy(self.__dict__).items() if key in self.serialized_attributes}, - } - - @classmethod - def from_dict(cls: type[T], dct: dict) -> T: - dct.pop("dgst") - return cls(**dct) - - @abstractmethod - def compute_heuristics_version(self) -> None: - raise NotImplementedError("Not meant to be implemented") diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py deleted file mode 100644 index acd54178..00000000 --- a/sec_certs/sample/common_criteria.py +++ /dev/null @@ -1,986 +0,0 @@ -from __future__ import annotations - -import copy -import re -from collections import Counter, defaultdict -from dataclasses import dataclass, field -from datetime import date, datetime -from enum import Enum -from pathlib import Path -from typing import Any, ClassVar -from urllib.parse import unquote_plus, urlparse - -import numpy as np -import requests -from bs4 import Tag - -import sec_certs.utils.extract -import sec_certs.utils.pdf -import sec_certs.utils.sanitization -from sec_certs import constants as constants -from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, cc_rules, rules, security_level_csv_scan -from sec_certs.sample.cc_certificate_id import canonicalize -from sec_certs.sample.certificate import Certificate -from sec_certs.sample.certificate import Heuristics as BaseHeuristics -from sec_certs.sample.certificate import PdfData as BasePdfData -from sec_certs.sample.certificate import References, logger -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.sample.sar import SAR -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.utils import helpers -from sec_certs.utils.extract import normalize_match_string - -HEADERS = { - "anssi": sec_certs.utils.extract.search_only_headers_anssi, - "bsi": sec_certs.utils.extract.search_only_headers_bsi, - "nscib": sec_certs.utils.extract.search_only_headers_nscib, - "niap": sec_certs.utils.extract.search_only_headers_niap, - "canada": sec_certs.utils.extract.search_only_headers_canada, -} - - -class ReferenceType(Enum): - DIRECT = "direct" - INDIRECT = "indirect" - - -class CommonCriteriaCert( - Certificate["CommonCriteriaCert", "CommonCriteriaCert.Heuristics", "CommonCriteriaCert.PdfData"], - PandasSerializableType, - ComplexSerializableType, -): - """ - Data structure for common criteria certificate. Contains several inner classes that layer the data logic. - Can be serialized into/from json (`ComplexSerializableType`) or pandas (`PandasSerializableType)`. - Is basic element of `CCDataset`. The functionality is mostly related to holding data and transformations that - the certificate can handle itself. `CCDataset` class then instrument this functionality. - """ - - cc_url = "http://www.commoncriteriaportal.org" - empty_st_url = "http://www.commoncriteriaportal.org/files/epfiles/" - - @dataclass(eq=True, frozen=True) - class MaintenanceReport(ComplexSerializableType): - """ - Object for holding maintenance reports. - """ - - maintenance_date: date | None - maintenance_title: str | None - maintenance_report_link: str | None - maintenance_st_link: str | None - - def __post_init__(self): - super().__setattr__( - "maintenance_report_link", sec_certs.utils.sanitization.sanitize_link(self.maintenance_report_link) - ) - super().__setattr__( - "maintenance_st_link", sec_certs.utils.sanitization.sanitize_link(self.maintenance_st_link) - ) - super().__setattr__( - "maintenance_title", sec_certs.utils.sanitization.sanitize_string(self.maintenance_title) - ) - super().__setattr__("maintenance_date", sec_certs.utils.sanitization.sanitize_date(self.maintenance_date)) - - @classmethod - def from_dict(cls, dct: dict) -> CommonCriteriaCert.MaintenanceReport: - new_dct = dct.copy() - new_dct["maintenance_date"] = ( - date.fromisoformat(dct["maintenance_date"]) - if isinstance(dct["maintenance_date"], str) - else dct["maintenance_date"] - ) - return super().from_dict(new_dct) - - def __lt__(self, other): - return self.maintenance_date < other.maintenance_date - - @dataclass(init=False) - class InternalState(ComplexSerializableType): - """ - Holds internal state of the certificate, whether downloads and converts of individual components succeeded. Also - holds information about errors and paths to the files. - """ - - st_download_ok: bool # Whether target download went OK - report_download_ok: bool # Whether report download went OK - st_convert_garbage: bool # Whether initial target conversion resulted in garbage - report_convert_garbage: bool # Whether initial report conversion resulted in garbage - st_convert_ok: bool # Whether overall target conversion went OK (either pdftotext or via OCR) - report_convert_ok: bool # Whether overall report conversion went OK (either pdftotext or via OCR) - st_extract_ok: bool # Whether target extraction went OK - report_extract_ok: bool # Whether report extraction went OK - - st_pdf_hash: str | None - report_pdf_hash: str | None - st_txt_hash: str | None - report_txt_hash: str | None - - st_pdf_path: Path - report_pdf_path: Path - st_txt_path: Path - report_txt_path: Path - - def __init__( - self, - st_download_ok: bool = False, - report_download_ok: bool = False, - st_convert_garbage: bool = False, - report_convert_garbage: bool = False, - st_convert_ok: bool = False, - report_convert_ok: bool = False, - st_extract_ok: bool = False, - report_extract_ok: bool = False, - st_pdf_hash: str | None = None, - report_pdf_hash: str | None = None, - st_txt_hash: str | None = None, - report_txt_hash: str | None = None, - ): - super().__init__() - self.st_download_ok = st_download_ok - self.report_download_ok = report_download_ok - self.st_convert_garbage = st_convert_garbage - self.report_convert_garbage = report_convert_garbage - self.st_convert_ok = st_convert_ok - self.report_convert_ok = report_convert_ok - self.st_extract_ok = st_extract_ok - self.report_extract_ok = report_extract_ok - self.st_pdf_hash = st_pdf_hash - self.report_pdf_hash = report_pdf_hash - self.st_txt_hash = st_txt_hash - self.report_txt_hash = report_txt_hash - - @property - def serialized_attributes(self) -> list[str]: - return [ - "st_download_ok", - "report_download_ok", - "st_convert_garbage", - "report_convert_garbage", - "st_convert_ok", - "report_convert_ok", - "st_extract_ok", - "report_extract_ok", - "st_pdf_hash", - "report_pdf_hash", - "st_txt_hash", - "report_txt_hash", - ] - - def report_is_ok_to_download(self, fresh: bool = True) -> bool: - return True if fresh else not self.report_download_ok - - def st_is_ok_to_download(self, fresh: bool = True) -> bool: - return True if fresh else not self.st_download_ok - - def report_is_ok_to_convert(self, fresh: bool = True) -> bool: - return self.report_download_ok if fresh else self.report_download_ok and not self.report_convert_ok - - def st_is_ok_to_convert(self, fresh: bool = True) -> bool: - return self.st_download_ok if fresh else self.st_download_ok and not self.st_convert_ok - - def report_is_ok_to_analyze(self, fresh: bool = True) -> bool: - if fresh is True: - return self.report_download_ok and self.report_convert_ok - else: - return self.report_download_ok and self.report_convert_ok and not self.report_extract_ok - - def st_is_ok_to_analyze(self, fresh: bool = True) -> bool: - if fresh is True: - return self.st_download_ok and self.st_convert_ok - else: - return self.st_download_ok and self.st_convert_ok and not self.st_extract_ok - - @dataclass - class PdfData(BasePdfData, ComplexSerializableType): - """ - Class that holds data extracted from pdf files. - """ - - report_metadata: dict[str, Any] | None = field(default=None) - st_metadata: dict[str, Any] | None = field(default=None) - report_frontpage: dict[str, dict[str, Any]] | None = field(default=None) - st_frontpage: dict[str, dict[str, Any]] | None = field(default=None) - report_keywords: dict[str, Any] | None = field(default=None) - st_keywords: dict[str, Any] | None = field(default=None) - report_filename: str | None = field(default=None) - st_filename: str | None = field(default=None) - - def __bool__(self) -> bool: - return any([x is not None for x in vars(self)]) - - @property - def bsi_data(self) -> dict[str, Any] | None: - """ - Returns frontpage data related to BSI-provided information - """ - return self.report_frontpage.get("bsi", None) if self.report_frontpage else None - - @property - def niap_data(self) -> dict[str, Any] | None: - """ - Returns frontpage data related to niap-provided information - """ - return self.report_frontpage.get("niap", None) if self.report_frontpage else None - - @property - def nscib_data(self) -> dict[str, Any] | None: - """ - Returns frontpage data related to nscib-provided information - """ - return self.report_frontpage.get("nscib", None) if self.report_frontpage else None - - @property - def canada_data(self) -> dict[str, Any] | None: - """ - Returns frontpage data related to canada-provided information - """ - return self.report_frontpage.get("canada", None) if self.report_frontpage else None - - @property - def anssi_data(self) -> dict[str, Any] | None: - """ - Returns frontpage data related to ANSSI-provided information - """ - return self.report_frontpage.get("anssi", None) if self.report_frontpage else None - - @property - def cert_lab(self) -> list[str] | None: - """ - Returns labs for which certificate data was parsed. - """ - labs = [ - data["cert_lab"].split(" ")[0].upper() - for data in [self.bsi_data, self.anssi_data, self.niap_data, self.nscib_data, self.canada_data] - if data - ] - return labs if labs else None - - @property - def bsi_cert_id(self) -> str | None: - return self.bsi_data.get("cert_id", None) if self.bsi_data else None - - @property - def niap_cert_id(self) -> str | None: - return self.niap_data.get("cert_id", None) if self.niap_data else None - - @property - def nscib_cert_id(self) -> str | None: - return self.nscib_data.get("cert_id", None) if self.nscib_data else None - - @property - def canada_cert_id(self) -> str | None: - return self.canada_data.get("cert_id", None) if self.canada_data else None - - @property - def anssi_cert_id(self) -> str | None: - return self.anssi_data.get("cert_id", None) if self.anssi_data else None - - def frontpage_cert_id(self, scheme: str) -> dict[str, float]: - """ - Get cert_id candidate from the frontpage of the report. - """ - scheme_map = { - "DE": self.bsi_cert_id, - "US": self.niap_cert_id, - "NL": self.nscib_cert_id, - "CA": self.canada_cert_id, - "FR": self.anssi_cert_id, - } - if scheme in scheme_map and (candidate := scheme_map[scheme]): - return {candidate: 1.0} - return {} - - def filename_cert_id(self, scheme: str) -> dict[str, float]: - """ - Get cert_id candidates from the matches in the report filename. - """ - if not self.report_filename: - return {} - scheme_rules = rules["cc_cert_id"][scheme] - matches: Counter = Counter() - for rule in scheme_rules: - match = re.search(rule, self.report_filename) - if match: - cert_id = normalize_match_string(match.group()) - matches[cert_id] += 1 - if not matches: - return {} - total = max(matches.values()) - results = {} - for candidate, count in matches.items(): - results[candidate] = count / total - # TODO count length in weight - return results - - def keywords_cert_id(self, scheme: str) -> dict[str, float]: - """ - Get cert_id candidates from the keywords matches in the report. - """ - if not self.report_keywords: - return {} - cert_id_matches = self.report_keywords.get("cc_cert_id") - if not cert_id_matches: - return {} - - if scheme not in cert_id_matches: - return {} - matches: Counter = Counter(cert_id_matches[scheme]) - if not matches: - return {} - total = max(matches.values()) - results = {} - for candidate, count in matches.items(): - results[candidate] = count / total - # TODO count length in weight - return results - - def metadata_cert_id(self, scheme: str) -> dict[str, float]: - """ - Get cert_id candidates from the report metadata. - """ - scheme_rules = rules["cc_cert_id"][scheme] - fields = ("/Title", "/Subject") - matches: Counter = Counter() - for meta_field in fields: - field_val = self.report_metadata.get(meta_field) if self.report_metadata else None - if not field_val: - continue - for rule in scheme_rules: - match = re.search(rule, field_val) - if match: - cert_id = normalize_match_string(match.group()) - matches[cert_id] += 1 - if not matches: - return {} - total = max(matches.values()) - results = {} - for candidate, count in matches.items(): - results[candidate] = count / total - # TODO count length in weight - return results - - def candidate_cert_ids(self, scheme: str) -> dict[str, float]: - frontpage_id = self.frontpage_cert_id(scheme) - metadata_id = self.metadata_cert_id(scheme) - filename_id = self.filename_cert_id(scheme) - keywords_id = self.keywords_cert_id(scheme) - - # Join them and weigh them, each is normalized with weights from 0 to 1 (if anything is returned) - candidates: dict[str, float] = defaultdict(lambda: 0.0) - # TODO: Add heuristic based on ordering of ids (and extracted year + increment) - # TODO: Add heuristic based on length - for candidate, count in frontpage_id.items(): - candidates[canonicalize(candidate, scheme)] += count * 1.5 - for candidate, count in metadata_id.items(): - candidates[canonicalize(candidate, scheme)] += count * 1.2 - for candidate, count in keywords_id.items(): - candidates[canonicalize(candidate, scheme)] += count * 1.0 - for candidate, count in filename_id.items(): - candidates[canonicalize(candidate, scheme)] += count * 1.0 - return candidates - - @dataclass - class Heuristics(BaseHeuristics, ComplexSerializableType): - """ - Class for various heuristics related to CommonCriteriaCert - """ - - extracted_versions: set[str] | None = field(default=None) - cpe_matches: set[str] | None = field(default=None) - verified_cpe_matches: set[str] | None = field(default=None) - related_cves: set[str] | None = field(default=None) - cert_lab: list[str] | None = field(default=None) - cert_id: str | None = field(default=None) - st_references: References = field(default_factory=References) - report_references: References = field(default_factory=References) - extracted_sars: set[SAR] | None = field(default=None) - direct_transitive_cves: set[str] | None = field(default=None) - indirect_transitive_cves: set[str] | None = field(default=None) - - @property - def serialized_attributes(self) -> list[str]: - return copy.deepcopy(super().serialized_attributes) - - pandas_columns: ClassVar[list[str]] = [ - "dgst", - "cert_id", - "name", - "status", - "category", - "manufacturer", - "scheme", - "security_level", - "eal", - "not_valid_before", - "not_valid_after", - "report_link", - "st_link", - "cert_link", - "manufacturer_web", - "extracted_versions", - "cpe_matches", - "verified_cpe_matches", - "related_cves", - "directly_referenced_by", - "indirectly_referenced_by", - "directly_referencing", - "indirectly_referencing", - "extracted_sars", - "protection_profiles", - "cert_lab", - ] - - def __init__( - self, - status: str, - category: str, - name: str, - manufacturer: str | None, - scheme: str, - security_level: str | set[str], - not_valid_before: date | None, - not_valid_after: date | None, - report_link: str, - st_link: str, - cert_link: str | None, - manufacturer_web: str | None, - protection_profiles: set[ProtectionProfile] | None, - maintenance_updates: set[MaintenanceReport] | None, - state: InternalState | None, - pdf_data: PdfData | None, - heuristics: Heuristics | None, - ): - super().__init__() - - self.status = status - self.category = category - self.name = sec_certs.utils.sanitization.sanitize_string(name) - - self.manufacturer = None - if manufacturer: - self.manufacturer = sec_certs.utils.sanitization.sanitize_string(manufacturer) - - self.scheme = scheme - self.security_level = sec_certs.utils.sanitization.sanitize_security_levels(security_level) - self.not_valid_before = sec_certs.utils.sanitization.sanitize_date(not_valid_before) - self.not_valid_after = sec_certs.utils.sanitization.sanitize_date(not_valid_after) - self.report_link = sec_certs.utils.sanitization.sanitize_link(report_link) - self.st_link = sec_certs.utils.sanitization.sanitize_link(st_link) - self.cert_link = sec_certs.utils.sanitization.sanitize_link(cert_link) - self.manufacturer_web = sec_certs.utils.sanitization.sanitize_link(manufacturer_web) - self.protection_profiles = protection_profiles - self.maintenance_updates = maintenance_updates - self.state = self.InternalState() if not state else state - self.pdf_data = self.PdfData() if not pdf_data else pdf_data - self.heuristics: CommonCriteriaCert.Heuristics = self.Heuristics() if not heuristics else heuristics - - @property - def dgst(self) -> str: - """ - Computes the primary key of the sample using first 16 bytes of SHA-256 digest - """ - if not (self.name is not None and self.report_link is not None and self.category is not None): - raise RuntimeError("Certificate digest can't be computed, because information is missing.") - return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link) - - @property - def eal(self) -> str | None: - """ - Returns EAL of certificate if it was extracted, None otherwise. - """ - res = [x for x in self.security_level if re.match(security_level_csv_scan, x)] - if res and len(res) == 1: - return res[0] - if res and len(res) > 1: - raise ValueError(f"Expected single EAL in security_level field, got: {res}") - else: - if self.protection_profiles: - return helpers.choose_lowest_eal({x.pp_eal for x in self.protection_profiles if x.pp_eal}) - else: - return None - - @property - def actual_sars(self) -> set[SAR] | None: - """ - Computes actual SARs. First, SARs implied by EAL are computed. Then, these are augmented with heuristically extracted SARs - :return Optional[Set[SAR]]: Set of actual SARs of a certificate, None if empty - """ - sars = dict() - if self.eal: - sars = {x[0]: SAR(x[0], x[1]) for x in SARS_IMPLIED_FROM_EAL[self.eal[:4]]} - - if self.heuristics.extracted_sars: - for sar in self.heuristics.extracted_sars: - if sar not in sars or sar.level > sars[sar.family].level: - sars[sar.family] = sar - - return set(sars.values()) if sars else None - - @property - def label_studio_title(self) -> str | None: - return self.name - - @property - def pandas_tuple(self) -> tuple: - """ - Returns tuple of attributes meant for pandas serialization - """ - return ( - self.dgst, - self.heuristics.cert_id, - self.name, - self.status, - self.category, - self.manufacturer, - self.scheme, - self.security_level, - self.eal, - self.not_valid_before, - self.not_valid_after, - self.report_link, - self.st_link, - self.cert_link, - self.manufacturer_web, - self.heuristics.extracted_versions, - self.heuristics.cpe_matches, - self.heuristics.verified_cpe_matches, - self.heuristics.related_cves, - self.heuristics.report_references.directly_referenced_by, - self.heuristics.report_references.indirectly_referenced_by, - self.heuristics.report_references.directly_referencing, - self.heuristics.report_references.indirectly_referencing, - self.heuristics.extracted_sars, - [x.pp_name for x in self.protection_profiles] if self.protection_profiles else np.nan, - self.heuristics.cert_lab[0] if (self.heuristics.cert_lab and self.heuristics.cert_lab[0]) else np.nan, - ) - - def __str__(self) -> str: - printed_manufacturer = self.manufacturer if self.manufacturer else "Unknown manufacturer" - return str(printed_manufacturer) + " " + str(self.name) + " dgst: " + self.dgst - - def merge(self, other: CommonCriteriaCert, other_source: str | None = None) -> None: - """ - Merges with other CC sample. Assuming they come from different sources, e.g., csv and html. - Assuming that html source has better protection profiles, they overwrite CSV info - On other values the sanity checks are made. - """ - if self != other: - logger.warning( - f"Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}" - ) - - for att, val in vars(self).items(): - if not val: - setattr(self, att, getattr(other, att)) - elif other_source == "html" and att == "protection_profiles": - setattr(self, att, getattr(other, att)) - elif other_source == "html" and att == "maintenance_updates": - setattr(self, att, getattr(other, att)) - elif att == "state": - setattr(self, att, getattr(other, att)) - else: - if getattr(self, att) != getattr(other, att): - logger.warning( - f"When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}" - ) - - @classmethod - def from_dict(cls, dct: dict) -> CommonCriteriaCert: - """ - Deserializes dictionary into `CommonCriteriaCert` - """ - new_dct = dct.copy() - new_dct["maintenance_updates"] = set(dct["maintenance_updates"]) - new_dct["protection_profiles"] = set(dct["protection_profiles"]) - new_dct["not_valid_before"] = ( - date.fromisoformat(dct["not_valid_before"]) - if isinstance(dct["not_valid_before"], str) - else dct["not_valid_before"] - ) - new_dct["not_valid_after"] = ( - date.fromisoformat(dct["not_valid_after"]) - if isinstance(dct["not_valid_after"], str) - else dct["not_valid_after"] - ) - return super(cls, CommonCriteriaCert).from_dict(new_dct) - - @staticmethod - def _html_row_get_name(cell: Tag) -> str: - return list(cell.stripped_strings)[0] - - @staticmethod - def _html_row_get_manufacturer(cell: Tag) -> str | None: - if lst := list(cell.stripped_strings): - return lst[0] - else: - return None - - @staticmethod - def _html_row_get_scheme(cell: Tag) -> str: - return list(cell.stripped_strings)[0] - - @staticmethod - def _html_row_get_security_level(cell: Tag) -> set: - return set(cell.stripped_strings) - - @staticmethod - def _html_row_get_manufacturer_web(cell: Tag) -> str | None: - for link in cell.find_all("a"): - if link is not None and link.get("title") == "Vendor's web site" and link.get("href") != "http://": - return link.get("href") - return None - - @staticmethod - def _html_row_get_protection_profiles(cell: Tag) -> set: - protection_profiles = set() - for link in list(cell.find_all("a")): - if link.get("href") is not None and "/ppfiles/" in link.get("href"): - protection_profiles.add( - ProtectionProfile( - pp_name=str(link.contents[0]), pp_eal=None, pp_link=CommonCriteriaCert.cc_url + link.get("href") - ) - ) - return protection_profiles - - @staticmethod - def _html_row_get_date(cell: Tag) -> date | None: - text = cell.get_text() - extracted_date = datetime.strptime(text, "%Y-%m-%d").date() if text else None - return extracted_date - - @staticmethod - def _html_row_get_report_st_links(cell: Tag) -> tuple[str, str]: - links = cell.find_all("a") - assert links[1].get("title").startswith("Certification Report") - assert links[2].get("title").startswith("Security Target") - - report_link = CommonCriteriaCert.cc_url + links[1].get("href") - security_target_link = CommonCriteriaCert.cc_url + links[2].get("href") - - return report_link, security_target_link - - @staticmethod - def _html_row_get_cert_link(cell: Tag) -> str | None: - links = cell.find_all("a") - return CommonCriteriaCert.cc_url + links[0].get("href") if links else None - - @staticmethod - def _html_row_get_maintenance_div(cell: Tag) -> Tag | None: - divs = cell.find_all("div") - for d in divs: - if d.find("div") and d.stripped_strings and list(d.stripped_strings)[0] == "Maintenance Report(s)": - return d - return None - - @staticmethod - def _html_row_get_maintenance_updates(main_div: Tag) -> set[CommonCriteriaCert.MaintenanceReport]: - possible_updates = list(main_div.find_all("li")) - maintenance_updates = set() - for u in possible_updates: - text = list(u.stripped_strings)[0] - main_date = datetime.strptime(text.split(" ")[0], "%Y-%m-%d").date() if text else None - main_title = text.split("– ")[1] - main_report_link = None - main_st_link = None - links = u.find_all("a") - for link in links: - if link.get("title").startswith("Maintenance Report:"): - main_report_link = CommonCriteriaCert.cc_url + link.get("href") - elif link.get("title").startswith("Maintenance ST"): - main_st_link = CommonCriteriaCert.cc_url + link.get("href") - else: - logger.error("Unknown link in Maintenance part!") - maintenance_updates.add( - CommonCriteriaCert.MaintenanceReport(main_date, main_title, main_report_link, main_st_link) - ) - return maintenance_updates - - @classmethod - def from_html_row(cls, row: Tag, status: str, category: str) -> CommonCriteriaCert: - """ - Creates a CC sample from html row of commoncriteria.org webpage. - """ - - cells = list(row.find_all("td")) - if len(cells) != 7: - raise ValueError(f"Unexpected number of
elements in CC html row. Expected: 7, actual: {len(cells)}") - - name = CommonCriteriaCert._html_row_get_name(cells[0]) - manufacturer = CommonCriteriaCert._html_row_get_manufacturer(cells[1]) - manufacturer_web = CommonCriteriaCert._html_row_get_manufacturer_web(cells[1]) - scheme = CommonCriteriaCert._html_row_get_scheme(cells[6]) - security_level = CommonCriteriaCert._html_row_get_security_level(cells[5]) - protection_profiles = CommonCriteriaCert._html_row_get_protection_profiles(cells[0]) - not_valid_before = CommonCriteriaCert._html_row_get_date(cells[3]) - not_valid_after = CommonCriteriaCert._html_row_get_date(cells[4]) - report_link, st_link = CommonCriteriaCert._html_row_get_report_st_links(cells[0]) - cert_link = CommonCriteriaCert._html_row_get_cert_link(cells[2]) - maintenance_div = CommonCriteriaCert._html_row_get_maintenance_div(cells[0]) - maintenances = ( - CommonCriteriaCert._html_row_get_maintenance_updates(maintenance_div) if maintenance_div else set() - ) - - return cls( - status, - category, - name, - manufacturer, - scheme, - security_level, - not_valid_before, - not_valid_after, - report_link, - st_link, - cert_link, - manufacturer_web, - protection_profiles, - maintenances, - None, - None, - None, - ) - - def set_local_paths( - self, - report_pdf_dir: str | Path | None, - st_pdf_dir: str | Path | None, - report_txt_dir: str | Path | None, - st_txt_dir: str | Path | None, - ) -> None: - """ - Sets paths to files given the requested directories - - :param Optional[Union[str, Path]] report_pdf_dir: Directory where pdf reports shall be stored - :param Optional[Union[str, Path]] st_pdf_dir: Directory where pdf security targets shall be stored - :param Optional[Union[str, Path]] report_txt_dir: Directory where txt reports shall be stored - :param Optional[Union[str, Path]] st_txt_dir: Directory where txt security targets shall be stored - """ - if report_pdf_dir is not None: - self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf") - if st_pdf_dir is not None: - self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf") - if report_txt_dir is not None: - self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + ".txt") - if st_txt_dir is not None: - self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + ".txt") - - @staticmethod - def download_pdf_report(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Downloads pdf of certification report given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to download the pdf report for - :return CommonCriteriaCert: returns the modified certificate with updated state - """ - exit_code: str | int - if not cert.report_link: - exit_code = "No link" - else: - exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path) - if exit_code != requests.codes.ok: - error_msg = f"failed to download report from {cert.report_link}, code: {exit_code}" - logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - cert.state.report_download_ok = False - else: - cert.state.report_download_ok = True - cert.state.report_pdf_hash = helpers.get_sha256_filepath(cert.state.report_pdf_path) - cert.pdf_data.report_filename = unquote_plus(str(urlparse(cert.report_link).path).split("/")[-1]) - return cert - - @staticmethod - def download_pdf_st(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Downloads pdf of security target given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to download the pdf security target for - :return CommonCriteriaCert: returns the modified certificate with updated state - """ - exit_code: str | int - if not cert.st_link: - exit_code = "No link" - else: - exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path) - if exit_code != requests.codes.ok: - error_msg = f"failed to download ST from {cert.st_link}, code: {exit_code}" - logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - cert.state.st_download_ok = False - else: - cert.state.st_download_ok = True - cert.state.st_pdf_hash = helpers.get_sha256_filepath(cert.state.st_pdf_path) - cert.pdf_data.st_filename = unquote_plus(str(urlparse(cert.st_link).path).split("/")[-1]) - return cert - - @staticmethod - def convert_report_pdf(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Converts the pdf certification report to txt, given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to download the pdf report for - :return CommonCriteriaCert: the modified certificate with updated state - """ - ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file( - cert.state.report_pdf_path, cert.state.report_txt_path - ) - # If OCR was done the result was garbage - cert.state.report_convert_garbage = ocr_done - # And put the whole result into convert_ok - cert.state.report_convert_ok = ok_result - if not ok_result: - error_msg = "failed to convert report pdf->txt" - logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - else: - cert.state.report_txt_hash = helpers.get_sha256_filepath(cert.state.report_txt_path) - return cert - - @staticmethod - def convert_st_pdf(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Converts the pdf security target to txt, given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to download the pdf security target for - :return CommonCriteriaCert: the modified certificate with updated state - """ - ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path) - # If OCR was done the result was garbage - cert.state.st_convert_garbage = ocr_done - # And put the whole result into convert_ok - cert.state.st_convert_ok = ok_result - if not ok_result: - error_msg = "failed to convert security target pdf->txt" - logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - else: - cert.state.st_txt_hash = helpers.get_sha256_filepath(cert.state.st_txt_path) - return cert - - @staticmethod - def extract_st_pdf_metadata(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Extracts metadata from security target pdf given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to extract the metadata for. - :return CommonCriteriaCert: the modified certificate with updated state - """ - response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path) - if response != constants.RETURNCODE_OK: - cert.state.st_extract_ok = False - else: - cert.state.st_extract_ok = True - return cert - - @staticmethod - def extract_report_pdf_metadata(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Extracts metadata from certification report pdf given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to extract the metadata for. - :return CommonCriteriaCert: the modified certificate with updated state - """ - response, cert.pdf_data.report_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.report_pdf_path) - if response != constants.RETURNCODE_OK: - cert.state.report_extract_ok = False - else: - cert.state.report_extract_ok = True - return cert - - @staticmethod - def extract_st_pdf_frontpage(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Extracts data from security target pdf frontpage given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to extract the frontpage data for. - :return CommonCriteriaCert: the modified certificate with updated state - """ - cert.pdf_data.st_frontpage = {} - - for header_type, associated_header_func in HEADERS.items(): - response, cert.pdf_data.st_frontpage[header_type] = associated_header_func(cert.state.st_txt_path) - - if response != constants.RETURNCODE_OK: - cert.state.st_extract_ok = False - return cert - - @staticmethod - def extract_report_pdf_frontpage(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Extracts data from certification report pdf frontpage given the certificate. Staticmethod to allow for parallelization. - - :param CommonCriteriaCert cert: cert to extract the frontpage data for. - :return CommonCriteriaCert: the modified certificate with updated state - """ - cert.pdf_data.report_frontpage = {} - - for header_type, associated_header_func in HEADERS.items(): - response, cert.pdf_data.report_frontpage[header_type] = associated_header_func(cert.state.report_txt_path) - - if response != constants.RETURNCODE_OK: - cert.state.report_extract_ok = False - return cert - - @staticmethod - def extract_report_pdf_keywords(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Matches regular expresions in txt obtained from certification report and extracts the matches into attribute. - Static method to allow for parallelization - - :param CommonCriteriaCert cert: certificate to extract the keywords for. - :return CommonCriteriaCert: the modified certificate with extracted keywords. - """ - report_keywords = sec_certs.utils.extract.extract_keywords(cert.state.report_txt_path, cc_rules) - if report_keywords is None: - cert.state.report_extract_ok = False - else: - cert.pdf_data.report_keywords = report_keywords - return cert - - @staticmethod - def extract_st_pdf_keywords(cert: CommonCriteriaCert) -> CommonCriteriaCert: - """ - Matches regular expresions in txt obtained from security target and extracts the matches into attribute. - Static method to allow for parallelization - - :param CommonCriteriaCert cert: certificate to extract the keywords for. - :return CommonCriteriaCert: the modified certificate with extracted keywords. - """ - st_keywords = sec_certs.utils.extract.extract_keywords(cert.state.st_txt_path, cc_rules) - if st_keywords is None: - cert.state.st_extract_ok = False - else: - cert.pdf_data.st_keywords = st_keywords - return cert - - def compute_heuristics_version(self) -> None: - """ - Fills in the heuristically obtained version of certified product into attribute in heuristics class. - """ - self.heuristics.extracted_versions = helpers.compute_heuristics_version(self.name) if self.name else set() - - def compute_heuristics_cert_lab(self) -> None: - """ - Fills in the heuristically obtained evaluation laboratory into attribute in heuristics class. - """ - if not self.pdf_data: - logger.error("Cannot compute sample lab when pdf files were not processed.") - return - self.heuristics.cert_lab = self.pdf_data.cert_lab - - def compute_heuristics_cert_id(self): - """ - Compute the heuristics cert_id of this cert, using several methods. - - The candidate cert_ids are extracted from the frontpage, PDF metadata, filename, and keywords matches. - - Finally, the cert_id is canonicalized. - """ - if not self.pdf_data: - logger.warning("Cannot compute sample id when pdf files were not processed.") - return - # Extract candidate cert_ids - candidates = self.pdf_data.candidate_cert_ids(self.scheme) - - if candidates: - max_weight = max(candidates.values()) - max_candidates = list(filter(lambda x: candidates[x] == max_weight, candidates.keys())) - max_candidates.sort(key=len, reverse=True) - self.heuristics.cert_id = max_candidates[0] diff --git a/sec_certs/sample/cpe.py b/sec_certs/sample/cpe.py deleted file mode 100644 index a7532f7f..00000000 --- a/sec_certs/sample/cpe.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from functools import lru_cache -from typing import Any, ClassVar - -from sec_certs import constants -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.utils import helpers - - -@dataclass(init=False) -class CPE(PandasSerializableType, ComplexSerializableType): - uri: str - version: str - vendor: str - item_name: str - title: str | None - start_version: tuple[str, str] | None - end_version: tuple[str, str] | None - - __slots__ = ["uri", "version", "vendor", "item_name", "title", "start_version", "end_version"] - - pandas_columns: ClassVar[list[str]] = [ - "uri", - "vendor", - "item_name", - "version", - "title", - ] - - def __init__( - self, - uri: str, - title: str | None = None, - start_version: tuple[str, str] | None = None, - end_version: tuple[str, str] | None = None, - ): - super().__init__() - self.uri = uri - - splitted = helpers.split_unescape(self.uri, ":") - self.vendor = " ".join(splitted[3].split("_")) - self.item_name = " ".join(splitted[4].split("_")) - self.version = self.normalize_version(" ".join(splitted[5].split("_"))) - self.title = title - self.start_version = start_version - self.end_version = end_version - - def __lt__(self, other: CPE) -> bool: - return self.uri < other.uri - - @staticmethod - def normalize_version(version: str) -> str: - """ - Maps common empty versions (empty '', asterisk '*') to unified empty version (constants.CPE_VERSION_NA) - """ - if version in {"", "*"}: - return constants.CPE_VERSION_NA - return version - - @classmethod - def from_dict(cls, dct: dict[str, Any]) -> CPE: - if isinstance(dct["start_version"], list): - dct["start_version"] = tuple(dct["start_version"]) - if isinstance(dct["end_version"], list): - dct["end_version"] = tuple(dct["end_version"]) - return super().from_dict(dct) - - @property - def serialized_attributes(self) -> list[str]: - return ["uri", "title", "start_version", "end_version"] - - @property - def update(self) -> str: - if self.uri is None: - raise RuntimeError("URI is missing.") - return " ".join(self.uri.split(":")[6].split("_")) - - @property - def target_hw(self) -> str: - if self.uri is None: - raise RuntimeError("URI is missing.") - return " ".join(self.uri.split(":")[10].split("_")) - - @property - def pandas_tuple(self) -> tuple: - return self.uri, self.vendor, self.item_name, self.version, self.title - - def __hash__(self) -> int: - return hash((self.uri, self.start_version, self.end_version)) - - def __eq__(self, other: object) -> bool: - return isinstance(other, self.__class__) and self.uri == other.uri - - -@lru_cache(maxsize=4096) -def cached_cpe(*args, **kwargs): - return CPE(*args, **kwargs) diff --git a/sec_certs/sample/cve.py b/sec_certs/sample/cve.py deleted file mode 100644 index ec024d35..00000000 --- a/sec_certs/sample/cve.py +++ /dev/null @@ -1,185 +0,0 @@ -from __future__ import annotations - -import datetime -import itertools -from dataclasses import dataclass -from typing import Any, ClassVar - -from dateutil.parser import isoparse - -from sec_certs.sample.cpe import CPE, cached_cpe -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.serialization.pandas import PandasSerializableType - - -@dataclass(init=False) -class CVE(PandasSerializableType, ComplexSerializableType): - @dataclass(eq=True) - class Impact(ComplexSerializableType): - base_score: float - severity: str - exploitability_score: float - impact_score: float - - __slots__ = ["base_score", "severity", "exploitability_score", "impact_score"] - - @classmethod - def from_nist_dict(cls, dct: dict[str, Any]) -> CVE.Impact: - """ - Will load Impact from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1 - """ - if not dct["impact"]: - return cls(0, "", 0, 0) - elif "baseMetricV3" in dct["impact"]: - return cls( - dct["impact"]["baseMetricV3"]["cvssV3"]["baseScore"], - dct["impact"]["baseMetricV3"]["cvssV3"]["baseSeverity"], - dct["impact"]["baseMetricV3"]["exploitabilityScore"], - dct["impact"]["baseMetricV3"]["impactScore"], - ) - elif "baseMetricV2" in dct["impact"]: - return cls( - dct["impact"]["baseMetricV2"]["cvssV2"]["baseScore"], - dct["impact"]["baseMetricV2"]["severity"], - dct["impact"]["baseMetricV2"]["exploitabilityScore"], - dct["impact"]["baseMetricV2"]["impactScore"], - ) - raise ValueError("NIST Dict for CVE Impact badly formatted.") - - cve_id: str - vulnerable_cpes: list[CPE] - impact: Impact - published_date: datetime.datetime | None - cwe_ids: set[str] | None - - __slots__ = ["cve_id", "vulnerable_cpes", "impact", "published_date", "cwe_ids"] - - pandas_columns: ClassVar[list[str]] = [ - "cve_id", - "vulnerable_cpes", - "base_score", - "severity", - "explotability_score", - "impact_score", - "published_date", - "cwe_ids", - ] - - def __init__( - self, cve_id: str, vulnerable_cpes: list[CPE], impact: Impact, published_date: str, cwe_ids: set[str] | None - ): - super().__init__() - self.cve_id = cve_id - self.vulnerable_cpes = vulnerable_cpes - self.impact = impact - self.published_date = isoparse(published_date) - self.cwe_ids = cwe_ids - - def __hash__(self) -> int: - return hash(self.cve_id) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, CVE): - return False - return self.cve_id == other.cve_id - - def __lt__(self, other: object) -> bool: - if not isinstance(other, CVE): - raise ValueError(f"Cannot compare CVE with {type(other)} type.") - self_year = int(self.cve_id.split("-")[1]) - self_id = int(self.cve_id.split("-")[2]) - other_year = int(other.cve_id.split("-")[1]) - other_id = int(other.cve_id.split("-")[2]) - - return self_year < other_year if self_year != other_year else self_id < other_id - - @property - def pandas_tuple(self): - return ( - self.cve_id, - self.vulnerable_cpes, - self.impact.base_score, - self.impact.severity, - self.impact.exploitability_score, - self.impact.impact_score, - self.published_date, - self.cwe_ids, - ) - - def to_dict(self) -> dict[str, Any]: - return { - "cve_id": self.cve_id, - "vulnerable_cpes": self.vulnerable_cpes, - "impact": self.impact, - "published_date": self.published_date.isoformat() if self.published_date else None, - "cwe_ids": self.cwe_ids, - } - - @staticmethod - def _parse_nist_dict(lst: list) -> list[CPE]: - cpes: list[CPE] = [] - - for x in lst: - if x["vulnerable"]: - cpe_uri = x["cpe23Uri"] - version_start: tuple[str, str] | None - version_end: tuple[str, str] | None - if "versionStartIncluding" in x and x["versionStartIncluding"]: - version_start = ("including", x["versionStartIncluding"]) - elif "versionStartExcluding" in x and x["versionStartExcluding"]: - version_start = ("excluding", x["versionStartExcluding"]) - else: - version_start = None - - if "versionEndIncluding" in x and x["versionEndIncluding"]: - version_end = ("including", x["versionEndIncluding"]) - elif "versionEndExcluding" in x and x["versionEndExcluding"]: - version_end = ("excluding", x["versionEndExcluding"]) - else: - version_end = None - - cpes.append(cached_cpe(cpe_uri, start_version=version_start, end_version=version_end)) - - return cpes - - @classmethod - def from_nist_dict(cls, dct: dict) -> CVE: - """ - Will load CVE from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1 - """ - - def get_vulnerable_cpes_from_nist_dict(dct: dict) -> list[CPE]: - def get_vulnerable_cpes_from_node(node: dict) -> list[CPE]: - cpes: list[CPE] = [] - - if node["operator"] == "AND": - return cpes - - if "children" in node: - for child in node["children"]: - cpes += get_vulnerable_cpes_from_node(child) - - if "cpe_match" not in node: - return cpes - - candidates = node["cpe_match"] - cpes += CVE._parse_nist_dict(candidates) - - return cpes - - return list( - itertools.chain.from_iterable(get_vulnerable_cpes_from_node(x) for x in dct["configurations"]["nodes"]) - ) - - cve_id = dct["cve"]["CVE_data_meta"]["ID"] - impact = cls.Impact.from_nist_dict(dct) - vulnerable_cpes = get_vulnerable_cpes_from_nist_dict(dct) - published_date = dct["publishedDate"] - cwe_ids = cls.parse_cwe_data(dct) - - return cls(cve_id, vulnerable_cpes, impact, published_date, cwe_ids) - - @staticmethod - def parse_cwe_data(dct: dict) -> set[str] | None: - descriptions = dct["cve"]["problemtype"]["problemtype_data"][0]["description"] - return {x["value"] for x in descriptions} if descriptions else None diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py deleted file mode 100644 index 770b21b0..00000000 --- a/sec_certs/sample/fips.py +++ /dev/null @@ -1,649 +0,0 @@ -from __future__ import annotations - -import itertools -import re -from dataclasses import dataclass, field -from datetime import date -from pathlib import Path -from typing import Any, Callable, ClassVar, Final, Literal - -import dateutil -import numpy as np -import requests -from bs4 import BeautifulSoup, Tag -from tabula import read_pdf - -import sec_certs.constants as constants -import sec_certs.utils.extract -import sec_certs.utils.helpers as helpers -import sec_certs.utils.pdf -import sec_certs.utils.pdf as pdf -import sec_certs.utils.tables as tables -from sec_certs.cert_rules import FIPS_ALGS_IN_TABLE, fips_rules -from sec_certs.config.configuration import config -from sec_certs.sample.certificate import Certificate -from sec_certs.sample.certificate import Heuristics as BaseHeuristics -from sec_certs.sample.certificate import PdfData as BasePdfData -from sec_certs.sample.certificate import References, logger -from sec_certs.sample.cpe import CPE -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.utils.helpers import fips_dgst - - -class FIPSHTMLParser: - def __init__(self, soup: BeautifulSoup): - self._soup = soup - - def get_web_data_and_algorithms(self) -> tuple[set[str], FIPSCertificate.WebData]: - divs = self._soup.find_all("div", class_="panel panel-default") - details_div, vendor_div, related_files_div, validation_history_div = divs - details_dict = self._build_details_dict(details_div) - - vendor_dict = self._build_vendor_dict(vendor_div) - related_files_dict = self._build_related_files_dict(related_files_div) - validation_history_dict = self._build_validation_history_dict(validation_history_div) - - algorithms = set() - if "algorithms" in details_dict: - algorithms_data = details_dict.pop("algorithms") - for category, alg_ids in algorithms_data.items(): - algorithms |= {category + x for x in alg_ids} - - return algorithms, FIPSCertificate.WebData( - **{**details_dict, **vendor_dict, **related_files_dict, **validation_history_dict} - ) - - def _build_details_dict(self, details_div: Tag) -> dict[str, Any]: - def parse_single_detail_entry(key, entry): - normalized_key = DETAILS_KEY_NORMALIZATION_DICT[key] - normalization_func = DETAILS_KEY_TO_NORMALIZATION_FUNCTION.get(normalized_key, None) - normalized_entry = ( - FIPSHTMLParser.normalize_string(entry.text) if not normalization_func else normalization_func(entry) - ) - return normalized_key, normalized_entry - - entries = details_div.find_all("div", class_="row padrow") - entries = zip( - [x.find("div", class_="col-md-3") for x in entries], [x.find("div", class_="col-md-9") for x in entries] - ) - entries = [(FIPSHTMLParser.normalize_string(key.text), entry) for key, entry in entries] - entries = [parse_single_detail_entry(*x) for x in entries if x[0] in DETAILS_KEY_NORMALIZATION_DICT.keys()] - entries = {x: y for x, y in entries} - - if "caveat" in entries: - entries["mentioned_certs"] = FIPSHTMLParser.get_mentioned_certs_from_caveat(entries["caveat"]) - - # Temporarily disabled, as this isn't extracting anything useful. Only UNKNOWN#1-9 algs were extracted over whole dataset. - # if "description" in entries: - # algs = FIPSHTMLParser.get_algs_from_description(entries["description"]) - # if "algorithms" in entries: - # entries["algorithms"].update({"UNKNOWN": x for x in algs}) - # else: - # entries["algorithms"] = {"UNKNOWN": x for x in algs} - - return entries - - @staticmethod - def _build_vendor_dict(vendor_div: Tag) -> dict[str, Any]: - if not (link := vendor_div.find("a")): - return {"vendor_url": None, "vendor": list(vendor_div.find("div", "panel-body").children)[0].strip()} - else: - return {"vendor_url": link.get("href"), "vendor": link.text.strip()} - - @staticmethod - def _build_related_files_dict(related_files_div: Tag) -> dict[str, Any]: - if cert_link := [x for x in related_files_div.find_all("a") if "Certificate" in x.text]: - return {"certificate_pdf_url": constants.FIPS_BASE_URL + cert_link[0].get("href")} - else: - return {"certificate_pdf_url": None} - - @staticmethod - def _build_validation_history_dict(validation_history_div: Tag) -> dict[str, Any]: - def parse_row(row): - validation_date, validation_type, lab = row.find_all("td") - return FIPSCertificate.ValidationHistoryEntry( - dateutil.parser.parse(validation_date.text).date(), validation_type.text, lab.text - ) - - rows = validation_history_div.find("tbody").find_all("tr") - history: list[FIPSCertificate.ValidationHistoryEntry] | None = [parse_row(x) for x in rows] if rows else None - return {"validation_history": history} - - @staticmethod - def get_mentioned_certs_from_caveat(caveat: str) -> dict[str, int]: - ids_found: dict[str, int] = {} - r_key = r"(?P\w+)?\s?(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)+(?P\d+)" - for m in re.finditer(r_key, caveat): - if m.group("word") and m.group("word").lower() in {"rsa", "shs", "dsa", "pkcs", "aes"}: - continue - if m.group("id") in ids_found: - ids_found[m.group("id")] += 1 - else: - ids_found[m.group("id")] = 1 - return ids_found - - @staticmethod - def get_algs_from_description(description: str) -> set[str]: - return {m.group() for m in re.finditer(FIPS_ALGS_IN_TABLE, description)} - - @staticmethod - def parse_algorithms(algorithms_div: Tag) -> dict[str, set[str]]: - rows = algorithms_div.find("tbody").find_all("tr") - dct: dict[str, set[str]] = dict() - for row in rows: - cells = row.find_all("td") - dct[cells[0].text] = {m.group() for m in re.finditer(FIPS_ALGS_IN_TABLE, cells[1].text)} - return dct - - @staticmethod - def normalize_string(string: str) -> str: - return " ".join(string.split()) - - @staticmethod - def parse_tested_configurations(tested_configurations: Tag) -> list[str] | None: - configurations = [y.text for y in tested_configurations.find_all("li")] - return configurations if not configurations == ["N/A"] else None - - @staticmethod - def normalize_embodiment(embodiment_element: Tag) -> str: - text = FIPSHTMLParser.normalize_string(embodiment_element.text) - embodiment_normalization_dict = { - "Multi-chip embedded": "Multi-Chip Embedded", - "Multi-chip Standalone": "Multi-Chip Stand Alone", - "Multi-chip standalone": "Multi-Chip Stand Alone", - "Single-chip": "Single Chip", - } - return embodiment_normalization_dict.get(text, text) - - -DETAILS_KEY_NORMALIZATION_DICT: Final[dict[str, str]] = { - "Module Name": "module_name", - "Standard": "standard", - "Status": "status", - "Sunset Date": "date_sunset", - "Validation Dates": "date_validation", - "Overall Level": "level", - "Caveat": "caveat", - "Security Level Exceptions": "exceptions", - "Module Type": "module_type", - "Embodiment": "embodiment", - "Approved Algorithms": "algorithms", - "Tested Configuration(s)": "tested_conf", - "Description": "description", - "Historical Reason": "historical_reason", - "Hardware Versions": "hw_versions", - "Firmware Versions": "fw_versions", - "Revoked Reason": "revoked_reason", - "Revoked Link": "revoked_link", - "Software Versions": "sw_versions", - "Product URL": "product_url", -} - -DETAILS_KEY_TO_NORMALIZATION_FUNCTION: dict[str, Callable] = { - "date_sunset": lambda x: dateutil.parser.parse(x.text).date(), - "algorithms": getattr(FIPSHTMLParser, "parse_algorithms"), - "tested_conf": getattr(FIPSHTMLParser, "parse_tested_configurations"), - "exceptions": lambda x: [y.text for y in x.find_all("li")], - "status": lambda x: FIPSHTMLParser.normalize_string(x.text).lower(), - "level": lambda x: int(FIPSHTMLParser.normalize_string(x.text)), - "embodiment": getattr(FIPSHTMLParser, "normalize_embodiment"), -} - - -class FIPSCertificate( - Certificate["FIPSCertificate", "FIPSCertificate.Heuristics", "FIPSCertificate.PdfData"], - PandasSerializableType, - ComplexSerializableType, -): - """ - Data structure for common FIPS 140 certificate. Contains several inner classes that layer the data logic. - Can be serialized into/from json (`ComplexSerializableType`). - Is basic element of `FIPSDataset`. The functionality is mostly related to holding data and transformations that - the certificate can handle itself. `FIPSDataset` class then instrument this functionality. - """ - - pandas_columns: ClassVar[list[str]] = [ - "dgst", - "cert_id", - "name", - "status", - "standard", - "type", - "level", - "embodiment", - "date_validation", - "date_sunset", - "algorithms", - "extracted_versions", - "cpe_matches", - "verified_cpe_matches", - "related_cves", - "module_directly_referenced_by", - "module_indirectly_referenced_by", - "module_directly_referencing", - "module_indirectly_referencing", - "policy_directly_referenced_by", - "policy_indirectly_referenced_by", - "policy_directly_referencing", - "policy_indirectly_referencing", - ] - - @dataclass(eq=True) - class InternalState(ComplexSerializableType): - """ - Holds state of the `FIPSCertificate` - """ - - module_download_ok: bool - policy_download_ok: bool - - policy_convert_garbage: bool - policy_convert_ok: bool - - module_extract_ok: bool - policy_extract_ok: bool - - policy_pdf_hash: str | None - policy_txt_hash: str | None - - policy_pdf_path: Path - policy_txt_path: Path - module_html_path: Path - - def __init__( - self, - module_download_ok: bool = False, - policy_download_ok: bool = False, - policy_convert_garbage: bool = False, - policy_convert_ok: bool = False, - module_extract_ok: bool = False, - policy_extract_ok: bool = False, - policy_pdf_hash: str | None = None, - policy_txt_hash: str | None = None, - ): - self.module_download_ok = module_download_ok - self.policy_download_ok = policy_download_ok - self.policy_convert_garbage = policy_convert_garbage - self.policy_convert_ok = policy_convert_ok - self.module_extract_ok = module_extract_ok - self.policy_extract_ok = policy_extract_ok - self.policy_pdf_hash = policy_pdf_hash - self.policy_txt_hash = policy_txt_hash - - @property - def serialized_attributes(self) -> list[str]: - return [ - "module_download_ok", - "policy_download_ok", - "policy_convert_garbage", - "policy_convert_ok", - "module_extract_ok", - "policy_extract_ok", - "policy_pdf_hash", - "policy_txt_hash", - ] - - def module_is_ok_to_download(self, fresh: bool = True) -> bool: - return True if fresh else not self.module_download_ok - - def policy_is_ok_to_download(self, fresh: bool = True) -> bool: - return True if fresh else not self.policy_download_ok - - def policy_is_ok_to_convert(self, fresh: bool = True) -> bool: - return self.policy_download_ok if fresh else self.policy_download_ok and not self.policy_convert_ok - - def module_is_ok_to_analyze(self, fresh: bool = True) -> bool: - return ( - self.module_download_ok and self.module_extract_ok - if fresh - else self.module_download_ok and not self.module_extract_ok - ) - - def policy_is_ok_to_analyze(self, fresh: bool = True) -> bool: - return ( - self.policy_convert_ok and self.policy_extract_ok - if fresh - else self.policy_convert_ok and not self.policy_extract_ok - ) - - def set_local_paths(self, policies_pdf_dir: Path, policies_txt_dir: Path, modules_html_dir: Path) -> None: - self.state.policy_pdf_path = (policies_pdf_dir / str(self.dgst)).with_suffix(".pdf") - self.state.policy_txt_path = (policies_txt_dir / str(self.dgst)).with_suffix(".txt") - self.state.module_html_path = (modules_html_dir / str(self.dgst)).with_suffix(".html") - - @dataclass(eq=True) - class ValidationHistoryEntry(ComplexSerializableType): - date: date - validation_type: Literal["initial", "update"] - lab: str - - @classmethod - def from_dict(cls, dct: dict) -> FIPSCertificate.ValidationHistoryEntry: - new_dct = dct.copy() - new_dct["date"] = dateutil.parser.parse(dct["date"]).date() - return cls(**new_dct) - - @dataclass(eq=True) - class WebData(ComplexSerializableType): - """ - Data structure for data obtained from scanning certificate webpage at NIST.gov - """ - - module_name: str | None = field(default=None) - validation_history: list[FIPSCertificate.ValidationHistoryEntry] | None = field(default=None) - vendor_url: str | None = field(default=None) - vendor: str | None = field(default=None) - certificate_pdf_url: str | None = field(default=None) - module_type: str | None = field(default=None) - standard: str | None = field(default=None) - status: Literal["active", "historical", "revoked"] | None = field(default=None) - level: Literal[1, 2, 3, 4] | None = field(default=None) - caveat: str | None = field(default=None) - exceptions: list[str] | None = field(default=None) - embodiment: str | None = field(default=None) - description: str | None = field(default=None) - tested_conf: list[str] | None = field(default=None) - hw_versions: str | None = field(default=None) - fw_versions: str | None = field(default=None) - sw_versions: str | None = field(default=None) - mentioned_certs: dict[str, int] | None = field(default=None) # Cert_id: n_occurences - historical_reason: str | None = field(default=None) - date_sunset: date | None = field(default=None) - revoked_reason: str | None = field(default=None) - revoked_link: str | None = field(default=None) - - # Those below are left unused at the moment - # product_url: Optional[str] = field(default=None) - - def __repr__(self) -> str: - return ( - self.module_name - if self.module_name is not None - else "" + " created by " + self.vendor - if self.vendor is not None - else "" - ) - - def __str__(self) -> str: - return repr(self) - - @classmethod - def from_dict(cls, dct: dict) -> FIPSCertificate.WebData: - new_dct = dct.copy() - if new_dct["date_sunset"]: - new_dct["date_sunset"] = dateutil.parser.parse(new_dct["date_sunset"]).date() - return cls(**dct) - - @dataclass(eq=True) - class PdfData(BasePdfData, ComplexSerializableType): - """ - Data structure that holds data obtained from scanning pdf files (or their converted txt documents). - """ - - keywords: dict = field(default_factory=dict) - policy_metadata: dict[str, Any] = field(default_factory=dict) - - @property - def certlike_algorithm_numbers(self) -> set[str]: - """Returns numbers of certificates from keywords["fips_certlike"]["Certlike"]""" - if self.keywords and "fips_certlike" in self.keywords: - fips_certlike = self.keywords["fips_certlike"].get("Certlike", dict()) - matches = {re.search(r"#\s{0,1}\d{1,4}", x) for x in fips_certlike.keys()} - return {"".join([x for x in match.group() if x.isdigit()]) for match in matches if match} - else: - return set() - - @dataclass(eq=True) - class Heuristics(BaseHeuristics, ComplexSerializableType): - """ - Data structure that holds data obtained by processing the certificate and applying various heuristics. - """ - - algorithms: set[str] = field(default_factory=set) - extracted_versions: set[str] = field(default_factory=set) - cpe_matches: set[str] | None = field(default=None) - verified_cpe_matches: set[CPE] | None = field(default=None) - related_cves: set[str] | None = field(default=None) - policy_prunned_references: set[str] = field(default_factory=set) - module_prunned_references: set[str] = field(default_factory=set) - policy_processed_references: References = field(default_factory=References) - module_processed_references: References = field(default_factory=References) - direct_transitive_cves: set[str] | None = field(default=None) - indirect_transitive_cves: set[str] | None = field(default=None) - - @property - def algorithm_numbers(self) -> set[str]: - """Returns numbers of algorithms""" - - def alg_to_number(alg: str) -> str: - return "".join([x for x in alg.split("#")[1] if x.isdigit()]) - - return {alg_to_number(x) for x in self.algorithms} - - @property - def dgst(self) -> str: - """ - Returns primary key of the certificate, its id. - """ - return fips_dgst(self.cert_id) - - @property - def manufacturer(self) -> str | None: # type: ignore - return self.web_data.vendor - - @property - def module_html_url(self) -> str: - return constants.FIPS_MODULE_URL.format(self.cert_id) - - @property - def policy_pdf_url(self) -> str: - return constants.FIPS_SP_URL.format(self.cert_id) - - @property - def name(self) -> str | None: # type: ignore - return self.web_data.module_name - - @property - def label_studio_title(self) -> str: - return ( - "Vendor: " - + str(self.web_data.vendor) - + "\n" - + "Module name: " - + str(self.web_data.module_name) - + "\n" - + "HW version: " - + str(self.web_data.hw_versions) - + "\n" - + "FW version: " - + str(self.web_data.fw_versions) - ) - - def __init__( - self, - cert_id: str, - web_data: FIPSCertificate.WebData | None = None, - pdf_data: FIPSCertificate.PdfData | None = None, - heuristics: FIPSCertificate.Heuristics | None = None, - state: InternalState | None = None, - ): - super().__init__() - - self.cert_id = cert_id - self.web_data: FIPSCertificate.WebData = web_data if web_data else FIPSCertificate.WebData() - self.pdf_data: FIPSCertificate.PdfData = pdf_data if pdf_data else FIPSCertificate.PdfData() - self.heuristics: FIPSCertificate.Heuristics = heuristics if heuristics else FIPSCertificate.Heuristics() - self.state: FIPSCertificate.InternalState = state if state else FIPSCertificate.InternalState() - - @property - def pandas_tuple(self) -> tuple: - return ( - self.dgst, - self.cert_id, - self.web_data.module_name, - self.web_data.status, - self.web_data.standard, - self.web_data.module_type, - self.web_data.level, - self.web_data.embodiment, - self.web_data.validation_history[0].date if self.web_data.validation_history else np.nan, - self.web_data.date_sunset, - self.heuristics.algorithms, - self.heuristics.extracted_versions, - self.heuristics.cpe_matches, - self.heuristics.verified_cpe_matches, - self.heuristics.related_cves, - self.heuristics.module_processed_references.directly_referenced_by, - self.heuristics.module_processed_references.indirectly_referenced_by, - self.heuristics.module_processed_references.directly_referencing, - self.heuristics.module_processed_references.indirectly_referencing, - self.heuristics.policy_processed_references.directly_referenced_by, - self.heuristics.policy_processed_references.indirectly_referenced_by, - self.heuristics.policy_processed_references.directly_referencing, - self.heuristics.policy_processed_references.indirectly_referencing, - ) - - @staticmethod - def parse_html_module(cert: FIPSCertificate) -> FIPSCertificate: - with cert.state.module_html_path.open("r") as handle: - soup = BeautifulSoup(handle, "html5lib") - - parser = FIPSHTMLParser(soup) - algorithms, cert.web_data = parser.get_web_data_and_algorithms() - cert.heuristics.algorithms |= algorithms - cert.state.module_extract_ok = True - - return cert - - @staticmethod - def download_module(cert: FIPSCertificate) -> FIPSCertificate: - if (exit_code := helpers.download_file(cert.module_html_url, cert.state.module_html_path)) != requests.codes.ok: - error_msg = f"failed to download html module from {cert.module_html_url}, code {exit_code}" - logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - cert.state.module_download_ok = False - else: - cert.state.module_download_ok = True - return cert - - @staticmethod - def download_policy(cert: FIPSCertificate) -> FIPSCertificate: - if (exit_code := helpers.download_file(cert.policy_pdf_url, cert.state.policy_pdf_path)) != requests.codes.ok: - error_msg = f"failed to download pdf policy from {cert.policy_pdf_url}, code {exit_code}" - logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - cert.state.policy_download_ok = False - else: - cert.state.policy_download_ok = True - cert.state.policy_pdf_hash = helpers.get_sha256_filepath(cert.state.policy_pdf_path) - return cert - - @staticmethod - def convert_policy_pdf(cert: FIPSCertificate) -> FIPSCertificate: - """ - Converts policy pdf -> txt - """ - ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file( - cert.state.policy_pdf_path, cert.state.policy_txt_path - ) - - # If OCR was done and the result was garbage - cert.state.policy_convert_garbage = ocr_done - # And put the whole result into convert_ok - cert.state.policy_convert_ok = ok_result - - if not ok_result: - error_msg = "Failed to convert policy pdf->txt" - logger.error(f"Cert dgst: {cert.dgst}" + error_msg) - else: - cert.state.policy_txt_hash = helpers.get_sha256_filepath(cert.state.policy_txt_path) - - return cert - - @staticmethod - def extract_policy_pdf_metadata(cert: FIPSCertificate) -> FIPSCertificate: - """ - Extract the PDF metadata from the security policy. - """ - _, metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.policy_pdf_path) - - if metadata: - cert.pdf_data.policy_metadata = metadata - else: - cert.pdf_data.policy_metadata = dict() - cert.state.policy_extract_ok = False - return cert - - @staticmethod - def extract_policy_pdf_keywords(cert: FIPSCertificate) -> FIPSCertificate: - """ - Extract keywords from policy document - """ - keywords = sec_certs.utils.extract.extract_keywords(cert.state.policy_txt_path, fips_rules) - if not keywords: - cert.state.policy_extract_ok = False - else: - cert.pdf_data.keywords = keywords - return cert - - @staticmethod - def get_algorithms_from_policy_tables(cert: FIPSCertificate): - """ - Retrieves IDs of algorithms from tables inside security policy pdfs. - External library is used to handle this. - """ - if table_rich_page_numbers := tables.find_pages_with_tables(cert.state.policy_txt_path): - pdf.repair_pdf(cert.state.policy_pdf_path) - try: - tabular_data = read_pdf(cert.state.policy_pdf_path, pages=list(table_rich_page_numbers), silent=True) - cert.heuristics.algorithms |= set( - itertools.chain.from_iterable(tables.get_algs_from_table(df.to_string()) for df in tabular_data) - ) - except Exception as e: - logger.warning(f"Error when parsing tables from {cert.dgst}: {e}") - cert.state.policy_extract_ok = False - - def prune_referenced_cert_ids(self) -> None: - """ - This method goes through all IDs (numbers) that correspond to FIPS Certificates and are stored in - pdf_data.keywords or web_data.mentioned_certs. It performs prunning of these attributes and fills attributes - heuristics.prunned_module_references and heuristics.prunned_policy_references. These variables are further - processed and Reference objects are created from them. - """ - html_module_ids = set(self.web_data.mentioned_certs.keys()) if self.web_data.mentioned_certs else set() - self.heuristics.module_prunned_references = self._prune_reference_ids_variable(html_module_ids) - - if self.pdf_data.keywords: - pdf_policy_ids = set(self.pdf_data.keywords["fips_cert_id"].get("Cert", dict()).keys()) - pdf_policy_ids = {"".join([y for y in x if y.isdigit()]) for x in pdf_policy_ids} - else: - pdf_policy_ids = set() - - self.heuristics.policy_prunned_references = self._prune_reference_ids_variable(pdf_policy_ids) - - def compute_heuristics_version(self) -> None: - """ - Heuristically computes the version of the product. - """ - versions_for_extraction = "" - if self.web_data.module_name: - versions_for_extraction += f" {self.web_data.module_name}" - if self.web_data.hw_versions: - versions_for_extraction += f" {self.web_data.hw_versions}" - if self.web_data.fw_versions: - versions_for_extraction += f" {self.web_data.fw_versions}" - self.heuristics.extracted_versions = helpers.compute_heuristics_version(versions_for_extraction) - - def _prune_reference_ids_variable(self, attribute_to_prune: set[str]) -> set[str]: - """ - Prunnes cert_ids from variable "attribute_to_prune", return result. Steps: - 0. Consider only ids != self.cert_id - 1. Consider only ids > config.always_false_positive_fips_cert_id_threshold - 2. Consider only ids s.t. they don't appear in self.heuristics.algorithms - 3. Consider only ids s.t. they don't appear in self.pdf_data.keywords["fips_certlike"]["Certlike"] - """ - prunned = {x for x in attribute_to_prune if x != self.cert_id} - prunned = {x for x in prunned if int(x) > config.always_false_positive_fips_cert_id_threshold} - prunned = {x for x in prunned if x not in self.heuristics.algorithm_numbers} - prunned = {x for x in prunned if x not in self.pdf_data.certlike_algorithm_numbers} - - return prunned diff --git a/sec_certs/sample/fips_algorithm.py b/sec_certs/sample/fips_algorithm.py deleted file mode 100644 index 16f19e64..00000000 --- a/sec_certs/sample/fips_algorithm.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import date -from typing import ClassVar - -from sec_certs import constants -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.serialization.pandas import PandasSerializableType - - -@dataclass(eq=True, frozen=True) -class FIPSAlgorithm(PandasSerializableType, ComplexSerializableType): - """ - Data structure for algorithm of `FIPSCertificate` - """ - - alg_number: str - algorithm_type: str - vendor: str - implementation_name: str - validation_date: date - - pandas_columns: ClassVar[list[str]] = [ - "dgst", - "alg_number", - "algorithm_type", - "vendor", - "implementation_name", - "validation_date", - ] - - @property - def pandas_tuple(self) -> tuple: - return ( - self.dgst, - self.alg_number, - self.algorithm_type, - self.vendor, - self.implementation_name, - self.validation_date, - ) - - @property - def dgst(self) -> str: - return f"{self.algorithm_type}{self.alg_number}" - - @property - def page_url(self) -> str: - return constants.FIPS_ALG_URL.format(self.algorithm_type, self.alg_number) diff --git a/sec_certs/sample/fips_iut.py b/sec_certs/sample/fips_iut.py deleted file mode 100644 index cb521ee4..00000000 --- a/sec_certs/sample/fips_iut.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import date, datetime -from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Iterator, Mapping - -import requests -from bs4 import BeautifulSoup, Tag - -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.utils.helpers import to_utc - - -@dataclass(frozen=True) -class IUTEntry(ComplexSerializableType): - module_name: str - vendor_name: str - standard: str - iut_date: date - - def to_dict(self) -> dict[str, str]: - return {**self.__dict__, "iut_date": self.iut_date.isoformat()} - - @classmethod - def from_dict(cls, dct: Mapping) -> IUTEntry: - return cls( - dct["module_name"], - dct["vendor_name"], - dct["standard"], - date.fromisoformat(dct["iut_date"]), - ) - - -@dataclass -class IUTSnapshot(ComplexSerializableType): - entries: set[IUTEntry] - timestamp: datetime - last_updated: date - displayed: int | None - not_displayed: int | None - total: int | None - - def __len__(self) -> int: - return len(self.entries) - - def __iter__(self) -> Iterator[IUTEntry]: - yield from self.entries - - def to_dict(self) -> dict[str, int | None | list[IUTEntry] | str]: - return { - "entries": list(self.entries), - "timestamp": self.timestamp.isoformat(), - "last_updated": self.last_updated.isoformat(), - "displayed": self.displayed, - "not_displayed": self.not_displayed, - "total": self.total, - } - - @classmethod - def from_dict(cls, dct: Mapping) -> IUTSnapshot: - return cls( - set(dct["entries"]), - datetime.fromisoformat(dct["timestamp"]), - date.fromisoformat(dct["last_updated"]), - dct["displayed"], - dct["not_displayed"], - dct["total"], - ) - - @classmethod - def from_page(cls, content: bytes, snapshot_date: datetime) -> IUTSnapshot: - """ - Get an IUT snapshot from a HTML dump of the FIPS website. - """ - if not content: - raise ValueError("Empty content in IUT.") - soup = BeautifulSoup(content, "html5lib") - tables = soup.find_all("table") - if len(tables) != 1: - raise ValueError("Not only a single table in IUT.") - - last_updated_elem = next( - filter( - lambda e: isinstance(e, Tag) and e.name == "p", - soup.find(id="content").next_siblings, - ) - ) - last_updated_text = str(last_updated_elem.string).strip() - last_updated = datetime.strptime(last_updated_text, "Last Updated: %m/%d/%Y").date() - table = tables[0].find("tbody") - lines = table.find_all("tr") - entries = { - IUTEntry( - str(line[0].string), - str(line[1].string), - str(line[2].string), - datetime.strptime(str(line[3].string), "%m/%d/%Y").date(), - ) - for line in map(lambda tr: tr.find_all("td"), lines) - } - - # Parse footer - footer = soup.find(id="IUTFooter") - displayed: int | None - not_displayed: int | None - total: int | None - - if footer: - footer_lines = footer.find_all("tr") - displayed = int(footer_lines[0].find_all("td")[1].text) - not_displayed = int(footer_lines[1].find_all("td")[1].text) - total = int(footer_lines[2].find_all("td")[1].text) - else: - displayed, not_displayed, total = (None, None, None) - - return cls( - entries=entries, - timestamp=snapshot_date, - last_updated=last_updated, - displayed=displayed, - not_displayed=not_displayed, - total=total, - ) - - @classmethod - def from_dump(cls, dump_path: str | Path, snapshot_date: datetime | None = None) -> IUTSnapshot: - """ - Get an IUT snapshot from a HTML file dump of the FIPS website. - """ - dump_path = Path(dump_path) - if snapshot_date is None: - try: - snapshot_date = to_utc(datetime.fromisoformat(dump_path.name[len("fips_iut_") : -len(".html")])) - except Exception: - raise ValueError("snapshot_date not given and could not be inferred from filename.") - with dump_path.open("rb") as f: - content = f.read() - return cls.from_page(content, snapshot_date) - - @classmethod - def from_web(cls) -> IUTSnapshot: - """ - Get an IUT snapshot from the FIPS website right now. - """ - iut_resp = requests.get(constants.FIPS_IUT_URL) - if iut_resp.status_code != 200: - raise ValueError(f"Getting IUT snapshot failed: {iut_resp.status_code}") - - snapshot_date = to_utc(datetime.now()) - return cls.from_page(iut_resp.content, snapshot_date) - - @classmethod - def from_web_latest(cls) -> IUTSnapshot: - """ - Get a IUT snapshot from seccerts.org. - """ - iut_resp = requests.get(config.fips_iut_latest_snapshot) - if iut_resp.status_code != 200: - raise ValueError(f"Getting MIP snapshot failed: {iut_resp.status_code}") - with NamedTemporaryFile() as tmpfile: - tmpfile.write(iut_resp.content) - return cls.from_json(tmpfile.name) diff --git a/sec_certs/sample/fips_mip.py b/sec_certs/sample/fips_mip.py deleted file mode 100644 index 6918d2aa..00000000 --- a/sec_certs/sample/fips_mip.py +++ /dev/null @@ -1,242 +0,0 @@ -from __future__ import annotations - -import logging -from dataclasses import dataclass -from datetime import date, datetime -from enum import Enum -from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Iterator, Mapping - -import requests -from bs4 import BeautifulSoup, Tag - -from sec_certs import constants -from sec_certs.config.configuration import config -from sec_certs.constants import FIPS_MIP_STATUS_RE -from sec_certs.serialization.json import ComplexSerializableType -from sec_certs.utils.helpers import to_utc - -logger = logging.getLogger(__name__) - - -class MIPStatus(Enum): - IN_REVIEW = "In Review" - REVIEW_PENDING = "Review Pending" - COORDINATION = "Coordination" - FINALIZATION = "Finalization" - - -@dataclass(frozen=True) -class MIPEntry(ComplexSerializableType): - module_name: str - vendor_name: str - standard: str - status: MIPStatus | None - status_since: date | None - - def to_dict(self) -> dict[str, str | MIPStatus | None | date | None]: - return { - **self.__dict__, - "status": self.status.value if self.status else None, - "status_since": self.status_since.isoformat() if self.status_since else None, - } - - @classmethod - def from_dict(cls, dct: Mapping) -> MIPEntry: - return cls( - dct["module_name"], - dct["vendor_name"], - dct["standard"], - MIPStatus(dct["status"]) if dct["status"] else None, - date.fromisoformat(dct["status_since"]) if dct.get("status_since") else None, - ) - - -@dataclass -class MIPSnapshot(ComplexSerializableType): - entries: set[MIPEntry] - timestamp: datetime - last_updated: date - displayed: int - not_displayed: int - total: int - - def __len__(self) -> int: - return len(self.entries) - - def __iter__(self) -> Iterator[MIPEntry]: - yield from self.entries - - def to_dict(self) -> dict[str, int | str | list[MIPEntry]]: - return { - "entries": list(self.entries), - "timestamp": self.timestamp.isoformat(), - "last_updated": self.last_updated.isoformat(), - "displayed": self.displayed, - "not_displayed": self.not_displayed, - "total": self.total, - } - - @classmethod - def from_dict(cls, dct: Mapping) -> MIPSnapshot: - return cls( - set(dct["entries"]), - datetime.fromisoformat(dct["timestamp"]), - date.fromisoformat(dct["last_updated"]), - dct["displayed"], - dct["not_displayed"], - dct["total"], - ) - - @classmethod - def _extract_entries_1(cls, lines): - """Works until 2020.10.28 (including).""" - entries = set() - for tr in lines: - tds = tr.find_all("td") - status = None - if "mip-highlight" in tds[-1]["class"]: - status = MIPStatus.FINALIZATION - elif "mip-highlight" in tds[-2]["class"]: - status = MIPStatus.COORDINATION - elif "mip-highlight" in tds[-3]["class"]: - status = MIPStatus.REVIEW_PENDING - elif "mip-highlight" in tds[-4]["class"]: - status = MIPStatus.IN_REVIEW - entries.add(MIPEntry(str(tds[0].string), str(tds[1].string), str(tds[2].string), status, None)) - return entries - - @classmethod - def _extract_entries_2(cls, lines): - """Works until 2021.04.20 (including).""" - return { - MIPEntry( - str(line[0].string), str(line[1].string), str(line[2].string), MIPStatus(str(line[3].string)), None - ) - for line in map(lambda tr: tr.find_all("td"), lines) - } - - @classmethod - def _extract_entries_3(cls, lines): - """Works until 2022.03.23 (including).""" - return { - MIPEntry( - str(line[0].string), - str(" ".join(line[1].find_all(text=True, recursive=False)).strip()), - str(line[2].string), - MIPStatus(str(line[3].string)), - None, - ) - for line in map(lambda tr: tr.find_all("td"), lines) - } - - @classmethod - def _extract_entries_4(cls, lines): - """Works now.""" - entries = set() - for line in map(lambda tr: tr.find_all("td"), lines): - module_name = str(line[0].string) - vendor_name = str(" ".join(line[1].find_all(text=True, recursive=False)).strip()) - standard = str(line[2].string) - status_line = FIPS_MIP_STATUS_RE.match(str(line[3].string)) - if status_line is None: - raise ValueError("Cannot parse MIP status line.") - status = MIPStatus(status_line.group("status")) - since = datetime.strptime(status_line.group("since"), "%m/%d/%Y").date() - entries.add(MIPEntry(module_name, vendor_name, standard, status, since)) - return entries - - @classmethod - def _extract_entries(cls, lines, snapshot_date): - if snapshot_date <= datetime(2020, 10, 28): - entries = cls._extract_entries_1(lines) - elif snapshot_date <= datetime(2021, 4, 20): - entries = cls._extract_entries_2(lines) - elif snapshot_date <= datetime(2022, 3, 23): - entries = cls._extract_entries_3(lines) - else: - entries = cls._extract_entries_4(lines) - return entries - - @classmethod - def from_page(cls, content: bytes, snapshot_date: datetime) -> MIPSnapshot: - """ - Get a MIP snapshot from a HTML dump of the FIPS website. - """ - if not content: - raise ValueError("Empty content in MIP.") - soup = BeautifulSoup(content, "html5lib") - tables = soup.find_all("table") - if len(tables) != 1: - raise ValueError("Not only a single table in MIP data.") - - # Parse Last Updated - last_updated_elem = next( - filter( - lambda e: isinstance(e, Tag) and e.name == "p", - soup.find(id="content").next_siblings, - ) - ) - last_updated_text = str(last_updated_elem.string).strip() - last_updated = datetime.strptime(last_updated_text, "Last Updated: %m/%d/%Y").date() - - # Parse entries - table = tables[0].find("tbody") - lines = table.find_all("tr") - entries = cls._extract_entries(lines, snapshot_date) - - # Parse footer - footer = soup.find(id="MIPFooter") - footer_lines = footer.find_all("tr") - displayed = int(footer_lines[0].find_all("td")[1].text) - not_displayed = int(footer_lines[1].find_all("td")[1].text) - total = int(footer_lines[2].find_all("td")[1].text) - - return cls( - entries=entries, - timestamp=snapshot_date, - last_updated=last_updated, - displayed=displayed, - not_displayed=not_displayed, - total=total, - ) - - @classmethod - def from_dump(cls, dump_path: str | Path, snapshot_date: datetime | None = None) -> MIPSnapshot: - """ - Get a MIP snapshot from a HTML file dump of the FIPS website. - """ - dump_path = Path(dump_path) - if snapshot_date is None: - try: - snapshot_date = to_utc(datetime.fromisoformat(dump_path.name[len("fips_mip_") : -len(".html")])) - except Exception: - raise ValueError("snapshot_date not given and could not be inferred from filename.") - with dump_path.open("rb") as f: - content = f.read() - return cls.from_page(content, snapshot_date) - - @classmethod - def from_web(cls) -> MIPSnapshot: - """ - Get a MIP snapshot from the FIPS website right now. - """ - mip_resp = requests.get(constants.FIPS_MIP_URL) - if mip_resp.status_code != 200: - raise ValueError(f"Getting MIP snapshot failed: {mip_resp.status_code}") - - snapshot_date = to_utc(datetime.now()) - return cls.from_page(mip_resp.content, snapshot_date) - - @classmethod - def from_web_latest(cls) -> MIPSnapshot: - """ - Get a MIP snapshot from seccerts.org. - """ - mip_resp = requests.get(config.fips_mip_latest_snapshot) - if mip_resp.status_code != 200: - raise ValueError(f"Getting MIP snapshot failed: {mip_resp.status_code}") - with NamedTemporaryFile() as tmpfile: - tmpfile.write(mip_resp.content) - return cls.from_json(tmpfile.name) diff --git a/sec_certs/sample/protection_profile.py b/sec_certs/sample/protection_profile.py deleted file mode 100644 index b7c2ec34..00000000 --- a/sec_certs/sample/protection_profile.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -import copy -import logging -from dataclasses import dataclass -from typing import Any - -import sec_certs.utils.sanitization as sanitization -from sec_certs.serialization.json import ComplexSerializableType - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class ProtectionProfile(ComplexSerializableType): - """ - Object for holding protection profiles. - """ - - pp_name: str - pp_eal: str | None - pp_link: str | None = None - pp_ids: frozenset[str] | None = None - - def __post_init__(self): - super().__setattr__("pp_name", sanitization.sanitize_string(self.pp_name)) - super().__setattr__("pp_link", sanitization.sanitize_link(self.pp_link)) - - @classmethod - def from_dict(cls, dct: dict[str, Any]) -> ProtectionProfile: - new_dct = copy.deepcopy(dct) - new_dct["pp_ids"] = frozenset(new_dct["pp_ids"]) if new_dct["pp_ids"] else None - return cls(*tuple(new_dct.values())) - - @classmethod - def from_old_api_dict(cls, dct: dict[str, Any]) -> ProtectionProfile: - pp_name = sanitization.sanitize_string(dct["csv_scan"]["cc_pp_name"]) - pp_link = sanitization.sanitize_link(dct["csv_scan"]["link_pp_document"]) - pp_ids = frozenset(dct["processed"]["cc_pp_csvid"]) if dct["processed"]["cc_pp_csvid"] else None - eal_set = sanitization.sanitize_security_levels(dct["csv_scan"]["cc_security_level"]) - - if not len(eal_set) <= 1: - raise ValueError("EAL field should have single value or should be empty.") - - eal_str = list(eal_set)[0] if eal_set else None - - return cls(pp_name, eal_str, pp_link, pp_ids) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, ProtectionProfile): - return False - return self.pp_name == other.pp_name and self.pp_link == other.pp_link - - def __lt__(self, other: ProtectionProfile) -> bool: - return self.pp_name < other.pp_name diff --git a/sec_certs/sample/sar.py b/sec_certs/sample/sar.py deleted file mode 100644 index 31359299..00000000 --- a/sec_certs/sample/sar.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import annotations - -import re -from dataclasses import dataclass -from typing import Any - -from sec_certs.serialization.json import ComplexSerializableType - -SAR_CLASS_MAPPING = { - "APE": "Protection Profile evaluation", - "ACE": "Protection Profile configuration evaluation", - "ASE": "Security Target evaluation", - "ADV": "Development", - "AGD": "Guidance documents", - "ALC": "Life-cycle support", - "ATE": "Tests", - "AVA": "Vulnerability assessment", - "ACO": "Comoposition", -} - -SAR_CLASSES = {x for x in SAR_CLASS_MAPPING} -SAR_DICT_KEY = "cc_sar" - - -@dataclass(frozen=True, eq=True) -class SAR(ComplexSerializableType): - family: str - level: int - - @property - def assurance_class(self): - return SAR_CLASS_MAPPING.get(self.family.split("_")[0], None) - - @classmethod - def from_string(cls, string: str) -> SAR: - if not cls.contains_level(string): - raise ValueError("SAR misses level integer") - if not cls.matches_re(string): - raise ValueError("SAR does not match any regular expression") - family = string.split(".")[0] - level = int(string.split(".")[1]) - return cls(family, level) - - @staticmethod - def contains_level(string: str) -> bool: - if len(string.split(".")) == 1: - return False - return True - - @staticmethod - def matches_re(string: str) -> bool: - return any( - [re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING] - ) - - def __lt__(self, other: Any) -> bool: - if not isinstance(other, SAR): - raise ValueError(f"cannot compare {type(other)} with SAR.") - return str(self) < str(other) diff --git a/sec_certs/serialization/__init__.py b/sec_certs/serialization/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sec_certs/serialization/json.py b/sec_certs/serialization/json.py deleted file mode 100644 index 69bbc2ff..00000000 --- a/sec_certs/serialization/json.py +++ /dev/null @@ -1,145 +0,0 @@ -from __future__ import annotations - -import copy -import json -from datetime import date -from functools import wraps -from pathlib import Path -from typing import Any, Callable, TypeVar - -from sec_certs import constants - -T = TypeVar("T", bound="ComplexSerializableType") - - -class SerializationError(Exception): - pass - - -class ComplexSerializableType: - __slots__: tuple[str] - - def __init__(self, *args, **kwargs): - pass - - # Ideally, the serialized_fields would be an class variable referencing itself, but that it virtually impossible - # to achieve without using metaclasses. Not to complicate the code, we choose instance variable. - @property - def serialized_attributes(self) -> list[str]: - if hasattr(self, "__slots__") and self.__slots__: - return list(self.__slots__) - return list(self.__dict__.keys()) - - def to_dict(self) -> dict[str, Any]: - if hasattr(self, "__slots__") and self.__slots__: - return { - key: copy.deepcopy(getattr(self, key)) for key in self.__slots__ if key in self.serialized_attributes - } - return {key: val for key, val in copy.deepcopy(self.__dict__).items() if key in self.serialized_attributes} - - @classmethod - def from_dict(cls: type[T], dct: dict) -> T: - try: - return cls(**dct) - except TypeError as e: - raise TypeError(f"Dict: {dct} on {cls.__mro__}") from e - - def to_json(self, output_path: str | Path | None = None) -> None: - if not output_path and (not hasattr(self, "json_path") or not self.json_path): # type: ignore - raise SerializationError( - f"The object {self} of type {self.__class__} does not have json_path attribute set but to_json() was called without an argument." - ) - if not output_path: - output_path = self.json_path # type: ignore - if self.json_path == constants.DUMMY_NONEXISTING_PATH: # type: ignore - raise SerializationError(f"json_path attribute for '{get_class_fullname(self)}' was not yet set.") - if hasattr(self, "root_dir") and self.root_dir == constants.DUMMY_NONEXISTING_PATH: # type: ignore - raise SerializationError(f"root_dir attribute for '{get_class_fullname(self)}' was not yet set.") - - if Path(output_path).is_dir(): # type: ignore - raise SerializationError("output path for json cannot be directory.") - - # false positive MyPy warning, cannot be None - with Path(output_path).open("w") as handle: # type: ignore - json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False) - - @classmethod - def from_json(cls: type[T], input_path: str | Path) -> T: - input_path = Path(input_path) - with input_path.open("r") as handle: - obj = json.load(handle, cls=CustomJSONDecoder) - return obj - - -# Decorator for serialization -def serialize(func: Callable): - @wraps(func) - def inner_func(*args, **kwargs): - if not args or not issubclass(type(args[0]), ComplexSerializableType): - raise ValueError( - "@serialize decorator is to be used only on instance methods of ComplexSerializableType child classes." - ) - - if hasattr(args[0], "_root_dir") and args[0]._root_dir == constants.DUMMY_NONEXISTING_PATH: - raise SerializationError( - "The invoked method requires dataset serialization. Cannot serialize without root_dir set. You can set it with obj.root_dir = ..." - ) - - update_json = kwargs.pop("update_json", True) - result = func(*args, **kwargs) - if update_json: - args[0].to_json() - return result - - return inner_func - - -def get_class_fullname(obj: Any) -> str: - if isinstance(obj, type): - klass = obj - else: - klass = obj.__class__ - module = klass.__module__ - if module == "builtins": - return klass.__qualname__ - return module + "." + klass.__qualname__ - - -class CustomJSONEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, ComplexSerializableType): - return {**{"_type": get_class_fullname(obj)}, **obj.to_dict()} - if isinstance(obj, dict): - return obj - if isinstance(obj, set): - return {"_type": "Set", "elements": sorted(list(obj))} - if isinstance(obj, frozenset): - return sorted(list(obj)) - if isinstance(obj, date): - return str(obj) - if isinstance(obj, Path): - return str(obj) - return super().default(obj) - - -class CustomJSONDecoder(json.JSONDecoder): - """ - Custom JSONDecoder. Any complex object that should be de-serializable must inherit directly from class - ComplexSerializableType (nested inheritance does not currently work (because x.__subclassess__() prints only direct - subclasses. Any such class must implement methods to_dict() and from_dict(). These are used to drive serialization. - """ - - def __init__(self, *args, **kwargs): - json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) - self.serializable_complex_types = {get_class_fullname(x): x for x in ComplexSerializableType.__subclasses__()} - - def object_hook(self, obj): - if "_type" in obj and obj["_type"] == "Set": - return set(obj["elements"]) - if "_type" in obj and obj["_type"] in self.serializable_complex_types.keys(): - complex_type = obj.pop("_type") - return self.serializable_complex_types[complex_type].from_dict(obj) - elif "_type" in obj: - raise SerializationError(f"JSONDecoder doesn't know how to handle {obj}") - - return obj diff --git a/sec_certs/serialization/pandas.py b/sec_certs/serialization/pandas.py deleted file mode 100644 index dc0aae18..00000000 --- a/sec_certs/serialization/pandas.py +++ /dev/null @@ -1,16 +0,0 @@ -from abc import ABC, abstractmethod - - -class PandasSerializableType(ABC): - def __init__(self, *args, **kwargs): - pass - - @property - @abstractmethod - def pandas_tuple(self): - raise NotImplementedError("Not meant to be implemented") - - @property - @abstractmethod - def pandas_columns(self): - raise NotImplementedError("Not meant to be implemented") diff --git a/sec_certs/utils/__init__.py b/sec_certs/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sec_certs/utils/extract.py b/sec_certs/utils/extract.py deleted file mode 100644 index 09460fb7..00000000 --- a/sec_certs/utils/extract.py +++ /dev/null @@ -1,817 +0,0 @@ -from __future__ import annotations - -import logging -import os -import re -from collections import Counter -from enum import Enum -from pathlib import Path -from typing import Any, Iterator - -import numpy as np - -from sec_certs import constants as constants -from sec_certs.cert_rules import REGEXEC_SEP, cc_rules -from sec_certs.constants import FILE_ERRORS_STRATEGY, LINE_SEPARATOR, MAX_ALLOWED_MATCH_LENGTH - -logger = logging.getLogger(__name__) - - -def search_only_headers_anssi(filepath: Path): # noqa: C901 - # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! - class HEADER_TYPE(Enum): - HEADER_FULL = 1 - HEADER_MISSING_CERT_ITEM_VERSION = 2 - HEADER_MISSING_PROTECTION_PROFILES = 3 - HEADER_DUPLICITIES = 4 - - rules_certificate_preface = [ - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\\(s\\)(.+)Evaluation facility(.+)Recognition arrangements", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification report reference(.+)Product name \\(reference / version\\)(.+)TOE name \\(reference / version\\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", - ), - ( - HEADER_TYPE.HEADER_FULL, - "Certification report reference(.+)TOE name(.+)Product's reference/ version(.+)TOE's reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements", - ), - # corrupted text (duplicities) - ( - HEADER_TYPE.HEADER_DUPLICITIES, - "Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles", - ), - # rules without product version - ( - HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ( - HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - "Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - # rules without protection profile - ( - HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES, - "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", - ), - ] - - # statistics about rules success rate - num_rules_hits = {} - for rule in rules_certificate_preface: - num_rules_hits[rule[1]] = 0 - - items_found = {} # type: ignore # noqa - - try: - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath) - - # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs - pos = whole_text.find(" ") - if pos != -1: - pos = whole_text.find(" ", pos) - if pos != -1: - whole_text = whole_text[pos:] - - no_match_yet = True - other_rule_already_match = False - rule_index = -1 - for rule in rules_certificate_preface: - rule_index += 1 - rule_and_sep = rule[1] + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text): - if no_match_yet: - items_found[constants.TAG_HEADER_MATCH_RULES] = [] - no_match_yet = False - - # insert rule if at least one match for it was found - if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]: - items_found[constants.TAG_HEADER_MATCH_RULES].append(rule[1]) - - if not other_rule_already_match: - other_rule_already_match = True - else: - logger.warning(f"WARNING: multiple rules are matching same certification document: {filepath}") - - num_rules_hits[rule[1]] += 1 # add hit to this rule - match_groups = m.groups() - index_next_item = 0 - items_found[constants.TAG_CERT_ID] = normalize_match_string(match_groups[index_next_item]) - index_next_item += 1 - - items_found[constants.TAG_CERT_ITEM] = normalize_match_string(match_groups[index_next_item]) - index_next_item += 1 - - if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION: - items_found[constants.TAG_CERT_ITEM_VERSION] = "" - else: - items_found[constants.TAG_CERT_ITEM_VERSION] = normalize_match_string(match_groups[index_next_item]) - index_next_item += 1 - - if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES: - items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = "" - else: - items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( - match_groups[index_next_item] - ) - index_next_item += 1 - - items_found[constants.TAG_CC_VERSION] = normalize_match_string(match_groups[index_next_item]) - index_next_item += 1 - - items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(match_groups[index_next_item]) - index_next_item += 1 - - items_found[constants.TAG_DEVELOPER] = normalize_match_string(match_groups[index_next_item]) - index_next_item += 1 - - items_found[constants.TAG_CERT_LAB] = normalize_match_string(match_groups[index_next_item]) - index_next_item += 1 - except Exception as e: - relative_filepath = "/".join(str(filepath).split("/")[-4:]) - error_msg = f"Failed to parse ANSSI frontpage headers from {relative_filepath}; {e}" - logger.error(error_msg) - return error_msg, None - - # if True: - # print('# hits for rule') - # sorted_rules = sorted(num_rules_hits.items(), - # key=operator.itemgetter(1), reverse=True) - # used_rules = [] - # for rule in sorted_rules: - # print('{:4d} : {}'.format(rule[1], rule[0])) - # if rule[1] > 0: - # used_rules.append(rule[0]) - - return constants.RETURNCODE_OK, items_found - - -def search_only_headers_bsi(filepath: Path): # noqa: C901 - # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! - LINE_SEPARATOR_STRICT = " " - NUM_LINES_TO_INVESTIGATE = 15 - rules_certificate_preface = [ - "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)", - "(BSI-DSZ-CC-.+?) zu (.+?) der (.*)", - ] - - items_found = {} # type: ignore # noqa - no_match_yet = True - - try: - # Process front page with info: cert_id, certified_item and developer - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( - filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT - ) - - for rule in rules_certificate_preface: - rule_and_sep = rule + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text): - if no_match_yet: - items_found[constants.TAG_HEADER_MATCH_RULES] = [] - no_match_yet = False - - # insert rule if at least one match for it was found - if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]: - items_found[constants.TAG_HEADER_MATCH_RULES].append(rule) - - match_groups = m.groups() - cert_id = match_groups[0] - certified_item = match_groups[1] - developer = match_groups[2] - - FROM_KEYWORD_LIST = [" from ", " der "] - for from_keyword in FROM_KEYWORD_LIST: - from_keyword_len = len(from_keyword) - if certified_item.find(from_keyword) != -1: - logger.warning( - f"string {from_keyword} detected in certified item - shall not be here, fixing..." - ) - certified_item_first = certified_item[: certified_item.find(from_keyword)] - developer = certified_item[certified_item.find(from_keyword) + from_keyword_len :] - certified_item = certified_item_first - continue - - end_pos = developer.find("\f-") - if end_pos == -1: - end_pos = developer.find("\fBSI") - if end_pos == -1: - end_pos = developer.find("Bundesamt") - if end_pos != -1: - developer = developer[:end_pos] - - items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) - items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item) - items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer) - items_found[constants.TAG_CERT_LAB] = "BSI" - - # Process page with more detailed sample info - # PP Conformance, Functionality, Assurance - rules_certificate_third = ["PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified"] - - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath) - - for rule in rules_certificate_third: - rule_and_sep = rule + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text): - # check if previous rules had at least one match - if constants.TAG_CERT_ID not in items_found.keys(): - logger.error(f"ERROR: front page not found for file: {filepath}") - - match_groups = m.groups() - ref_protection_profiles = match_groups[0] - cc_version = match_groups[1] - cc_security_level = match_groups[2] - - items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( - ref_protection_profiles - ) - items_found[constants.TAG_CC_VERSION] = normalize_match_string(cc_version) - items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(cc_security_level) - - # print('\n*** Certificates without detected preface:') - # for file_name in files_without_match: - # print('No hits for {}'.format(file_name)) - # print('Total no hits files: {}'.format(len(files_without_match))) - # print('\n**********************************') - except Exception as e: - relative_filepath = "/".join(str(filepath).split("/")[-4:]) - error_msg = f"Failed to parse BSI headers from frontpage: {relative_filepath}; {e}" - logger.error(error_msg) - return error_msg, None - - return constants.RETURNCODE_OK, items_found - - -def search_only_headers_nscib(filepath: Path): # noqa: C901 - # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! - LINE_SEPARATOR_STRICT = " " - NUM_LINES_TO_INVESTIGATE = 60 - items_found: dict[str, str] = {} - - try: - # Process front page with info: cert_id, certified_item and developer - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( - filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT - ) - - certified_item = "" - developer = "" - cert_lab = "" - cert_id = "" - - lines = whole_text_with_newlines.splitlines() - no_match_yet = True - item_offset = -1 - - for line_index in range(0, len(lines)): - line = lines[line_index] - - if "Certification Report" in line: - item_offset = line_index + 1 - if "Assurance Continuity Maintenance Report" in line: - item_offset = line_index + 1 - - SPONSORDEVELOPER_STR = "Sponsor and developer:" - - if SPONSORDEVELOPER_STR in line: - if no_match_yet: - items_found = {} - no_match_yet = False - - # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report' - certified_item = "" - for name_index in range(item_offset, line_index): - certified_item += lines[name_index] + " " - developer = line[line.find(SPONSORDEVELOPER_STR) + len(SPONSORDEVELOPER_STR) :] - - SPONSOR_STR = "Sponsor:" - - if SPONSOR_STR in line: - if no_match_yet: - items_found = {} - no_match_yet = False - - # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report' - certified_item = "" - for name_index in range(item_offset, line_index): - certified_item += lines[name_index] + " " - - DEVELOPER_STR = "Developer:" - if DEVELOPER_STR in line: - developer = line[line.find(DEVELOPER_STR) + len(DEVELOPER_STR) :] - - CERTLAB_STR = "Evaluation facility:" - if CERTLAB_STR in line: - cert_lab = line[line.find(CERTLAB_STR) + len(CERTLAB_STR) :] - - REPORTNUM_STR = "Report number:" - if REPORTNUM_STR in line: - cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :] - - if not no_match_yet: - items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) - items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item) - items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer) - items_found[constants.TAG_CERT_LAB] = cert_lab - - except Exception as e: - error_msg = f"Failed to parse NSCIB headers from frontpage: {filepath}; {e}" - logger.error(error_msg) - return error_msg, None - - return constants.RETURNCODE_OK, items_found - - -def search_only_headers_niap(filepath: Path): - # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! - LINE_SEPARATOR_STRICT = " " - NUM_LINES_TO_INVESTIGATE = 15 - items_found: dict[str, str] = {} - - try: - # Process front page with info: cert_id, certified_item and developer - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( - filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT - ) - - certified_item = "" - cert_id = "" - - lines = whole_text_with_newlines.splitlines() - no_match_yet = True - item_offset = -1 - - for line_index in range(0, len(lines)): - line = lines[line_index] - - if "Validation Report" in line: - item_offset = line_index + 1 - - REPORTNUM_STR = "Report Number:" - if REPORTNUM_STR in line: - if no_match_yet: - items_found = {} - no_match_yet = False - - # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report' - certified_item = "" - for name_index in range(item_offset, line_index): - certified_item += lines[name_index] + " " - cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :] - break - - if not no_match_yet: - items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) - items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item) - items_found[constants.TAG_CERT_LAB] = "US NIAP" - - except Exception as e: - error_msg = f"Failed to parse NIAP headers from frontpage: {filepath}; {e}" - logger.error(error_msg) - return error_msg, None - - return constants.RETURNCODE_OK, items_found - - -def search_only_headers_canada(filepath: Path): # noqa: C901 - # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! - LINE_SEPARATOR_STRICT = " " - NUM_LINES_TO_INVESTIGATE = 20 - items_found: dict[str, str] = {} - try: - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( - filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT - ) - - cert_id = "" - - lines = whole_text_with_newlines.splitlines() - no_match_yet = True - for line_index in range(0, len(lines)): - line = lines[line_index] - if "Government of Canada, Communications Security Establishment" in line: - REPORTNUM_STR1 = "Evaluation number:" - REPORTNUM_STR2 = "Document number:" - matched_number_str = "" - line_certid = lines[line_index + 1] - if line_certid.startswith(REPORTNUM_STR1): - matched_number_str = REPORTNUM_STR1 - if line_certid.startswith(REPORTNUM_STR2): - matched_number_str = REPORTNUM_STR2 - if matched_number_str != "": - if no_match_yet: - items_found = {} - no_match_yet = False - - cert_id = line_certid[line_certid.find(matched_number_str) + len(matched_number_str) :] - break - - if ( - "Government of Canada. This document is the property of the Government of Canada. It shall not be altered," - in line - ): - REPORTNUM_STR = "Evaluation number:" - for offset in range(1, 20): - line_certid = lines[line_index + offset] - if "UNCLASSIFIED" in line_certid: - if no_match_yet: - items_found = {} - no_match_yet = False - line_certid = lines[line_index + offset - 4] - cert_id = line_certid[line_certid.find(REPORTNUM_STR) + len(REPORTNUM_STR) :] - break - if not no_match_yet: - break - - if ( - "UNCLASSIFIED / NON CLASSIFIÉ" in line - and "COMMON CRITERIA CERTIFICATION REPORT" in lines[line_index + 2] - ): - line_certid = lines[line_index + 1] - if no_match_yet: - items_found = {} - no_match_yet = False - cert_id = line_certid - break - - if not no_match_yet and cert_id: - items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) - items_found[constants.TAG_CERT_LAB] = "CANADA" - - except Exception as e: - error_msg = f"Failed to parse Canada headers from frontpage: {filepath}; {e}" - logger.error(error_msg) - return error_msg, None - - return constants.RETURNCODE_OK, items_found - - -def search_files(folder: str | Path) -> Iterator[str]: - for root, _, files in os.walk(str(folder)): - yield from [os.path.join(root, x) for x in files] - - -def flatten_matches(dct: dict) -> dict: - """ - Function to flatten dictionary of matches. - - Turns - ``` - {"a": {"cc": 3}, "b": {}, "d": {"dd": 4, "cc": 2}} - ``` - into - ``` - {"cc": 5, "dd": 4} - ``` - - :param dct: Dictionary to flatten - :return: Flattened dictionary - """ - result: Counter[Any] = Counter() - for key, value in dct.items(): - if isinstance(value, dict): - result.update(flatten_matches(value)) - else: - result[key] = value - return dict(result) - - -def prune_matches(dct: dict) -> dict: - """ - Prune a dictionary of matches. - - Turns - ``` - {"a": {"cc": 3}, "b": {"aa": {}, "bb": {}}, "d": {"dd": 4, "cc": 2}} - ``` - into - ``` - {"a": {"cc": 3}, "b": {}, "d": {"dd": 4, "cc": 2}} - ``` - - :param dct: The dictionary of matches. - :return: The pruned dictionary. - """ - - def walk(obj, depth): - if isinstance(obj, dict): - if not obj: - return None - res = {} - for k, v in obj.items(): - r = walk(v, depth + 1) - if r is not None: - res[k] = r - return res if res or depth == 1 else None - else: - return obj - - return walk(dct, 0) - - -def extract_keywords(filepath: Path, search_rules) -> dict[str, dict[str, int]] | None: - """ - Extract keywords from filepath using the search rules. - - :param filepath: - :param search_rules: - :return: - """ - - try: - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath, -1, LINE_SEPARATOR) - - def extract(rules): - if isinstance(rules, dict): - return {k: extract(v) for k, v in rules.items()} - elif isinstance(rules, list): - matches = [extract(rule) for rule in rules] - c = Counter() - for match_list in matches: - c += Counter(match_list) - return dict(c) - elif isinstance(rules, re.Pattern): - rule = rules - matches = [] - for match in rule.finditer(whole_text): - match = match.group("match") - match = normalize_match_string(match) - - match_len = len(match) - if match_len > MAX_ALLOWED_MATCH_LENGTH: - logger.warning(f"Excessive match with length of {match_len} detected for rule {rule.pattern}") - matches.append(match) - return matches - - result = extract(search_rules) - return prune_matches(result) - except Exception as e: - relative_filepath = "/".join(str(filepath).split("/")[-4:]) - error_msg = f"Failed to parse keywords from: {relative_filepath}; {e}" - logger.error(error_msg) - return None - - -def normalize_match_string(match: str) -> str: - match = match.strip().strip("[];.”\"':)(,").rstrip(os.sep).replace(" ", " ") - return "".join(filter(str.isprintable, match)) - - -def load_text_file( - file_name: str | Path, limit_max_lines: int = -1, line_separator: str = LINE_SEPARATOR -) -> tuple[str, str, bool]: - """ - Load the text contents of a file at `file_name`, upto `limit_max_lines` of lines, replace - newlines in the text with `line_separator`. - - :param file_name: The file_name to load. - :param limit_max_lines: The limit on number of lines to return. - :param line_separator: The string to replace newlines with. - :return: A tuple of three elements (the text with replaced newlines, the text and a boolean whether a unicode - decoding error happened). - """ - lines = [] - was_unicode_decode_error = False - with Path(file_name).open("r", errors=FILE_ERRORS_STRATEGY) as f: - try: - lines = f.readlines() - except UnicodeDecodeError: - was_unicode_decode_error = True - logger.warning("UnicodeDecodeError, opening as utf8") - - if was_unicode_decode_error: - with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2: - # coding failure, try line by line - line = " " - while line: - try: - line = f2.readline() - lines.append(line) - except UnicodeDecodeError: - # ignore error - continue - - whole_text = "" - whole_text_with_newlines = "" - lines_included = 0 - for line in lines: - if limit_max_lines != -1 and lines_included >= limit_max_lines: - break - - whole_text_with_newlines += line - line = line.replace("\n", "") - whole_text += line - whole_text += line_separator - lines_included += 1 - - return whole_text, whole_text_with_newlines, was_unicode_decode_error - - -def load_cert_html_file(file_name: str) -> str: - with open(file_name, errors=FILE_ERRORS_STRATEGY) as f: - try: - return f.read() - except UnicodeDecodeError: - logger.warning("UnicodeDecodeError, opening as utf8") - - with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2: - try: - return f2.read() - except UnicodeDecodeError: - logger.error(f"Failed to read file {file_name}") - return "" - - -def rules_get_subset(desired_path: str) -> dict: - """ - Recursively applies cc_certs.get(key) on tokens from desired_path, - returns the keys of the inner-most layer. - """ - dct = cc_rules - for token in desired_path.split("."): - dct = dct[token] - return dct - - -def extract_key_paths(dct: dict, current_path: str) -> list[str]: - """ - Given subset of cc_rules dictionary, will compute full paths to all leafs - in the dictionaries, s.t. the final value of each path is a list of regex - matches in the keywords dictionary. - """ - paths = [] - for key in dct: - if isinstance(dct[key], dict): - paths.extend(extract_key_paths(dct[key], current_path + "." + key)) - elif isinstance(dct[key], list): - paths.append(current_path + "." + key) - return paths - - -def get_sum_of_values_from_dict_path(dct: dict | None, path: str, default: float = np.nan) -> float: - """ - Given dictionary and path, will compute sum of occurences of values in the inner-most layer - of that path. If the key is missing from dict, return default value. - """ - if not dct: - return np.nan - - res = dct - - try: - for token in path.split("."): - res = res[token] - except KeyError: - return default - - return sum(res.values()) - - -def get_sums_for_rules_subset(dct: dict | None, path: str) -> dict[str, float]: - """ - Given path to search in cc_rules (e.g., "symmetric_crypto"), - will get the finest resolution and count occurences of the keys in the - examined dictionary. - """ - cc_rules_subset_to_search = rules_get_subset(path) - paths_to_search = extract_key_paths(cc_rules_subset_to_search, path) - return {x: get_sum_of_values_from_dict_path(dct, x, np.nan) for x in paths_to_search} diff --git a/sec_certs/utils/helpers.py b/sec_certs/utils/helpers.py deleted file mode 100644 index 302f4e6a..00000000 --- a/sec_certs/utils/helpers.py +++ /dev/null @@ -1,239 +0,0 @@ -from __future__ import annotations - -import hashlib -import logging -import re -import time -from contextlib import nullcontext -from datetime import datetime -from functools import partial -from pathlib import Path -from typing import Any, Collection - -import numpy as np -import pkgconfig -import requests - -import sec_certs.constants as constants -from sec_certs.config.configuration import config -from sec_certs.utils import parallel_processing -from sec_certs.utils.tqdm import tqdm - -logger = logging.getLogger(__name__) - - -def download_file( - url: str, output: Path, delay: float = 0, show_progress_bar: bool = False, progress_bar_desc: str | None = None -) -> str | int: - try: - time.sleep(delay) - # See https://github.com/psf/requests/issues/3953 for header justification - r = requests.get( - url, allow_redirects=True, timeout=constants.REQUEST_TIMEOUT, stream=True, headers={"Accept-Encoding": None} # type: ignore - ) - ctx: Any - if show_progress_bar: - ctx = partial( - tqdm, - total=int(r.headers.get("content-length", 0)), - unit="B", - unit_scale=True, - unit_divisor=1024, - desc=progress_bar_desc, - ) - else: - ctx = nullcontext - - if r.status_code == requests.codes.ok: - with ctx() as pbar: - with output.open("wb") as f: - for data in r.iter_content(1024): - f.write(data) - if show_progress_bar: - pbar.update(len(data)) - - return r.status_code - except requests.exceptions.Timeout: - return requests.codes.timeout - except Exception as e: - logger.error(f"Failed to download from {url}; {e}") - return constants.RETURNCODE_NOK - return constants.RETURNCODE_NOK - - -def download_parallel( - urls: Collection[str], paths: Collection[Path], progress_bar_desc: str | None = None -) -> list[int]: - exit_codes = parallel_processing.process_parallel( - download_file, list(zip(urls, paths)), config.n_threads, unpack=True, progress_bar_desc=progress_bar_desc - ) - n_successful = len([e for e in exit_codes if e == requests.codes.ok]) - logger.info(f"Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.") - - for url, e in zip(urls, exit_codes): - if e != requests.codes.ok: - logger.error(f"Failed to download {url}, exit code: {e}") - - return exit_codes - - -def fips_dgst(cert_id: int | str) -> str: - return get_first_16_bytes_sha256(str(cert_id)) - - -def get_first_16_bytes_sha256(string: str) -> str: - return hashlib.sha256(string.encode("utf-8")).hexdigest()[:16] - - -def get_sha256_filepath(filepath: str | Path) -> str: - hash_sha256 = hashlib.sha256() - with Path(filepath).open("rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_sha256.update(chunk) - return hash_sha256.hexdigest() - - -def to_utc(timestamp: datetime) -> datetime: - offset = timestamp.utcoffset() - if offset is None: - return timestamp - timestamp -= offset - timestamp = timestamp.replace(tzinfo=None) - return timestamp - - -def is_in_dict(target_dict: dict, path: str) -> bool: - current_level = target_dict - for item in path: - if item not in current_level: - return False - else: - current_level = current_level[item] - return True - - -def compute_heuristics_version(cert_name: str) -> set[str]: - """ - Will extract possible versions from the name of sample - """ - at_least_something = r"(\b(\d)+\b)" - just_numbers = r"(\d{1,5})(\.\d{1,5})" - - without_version = r"(" + just_numbers + r"+)" - long_version = r"(" + r"(\bversion)\s*" + just_numbers + r"+)" - short_version = r"(" + r"\bv\s*" + just_numbers + r"+)" - full_regex_string = r"|".join([without_version, short_version, long_version]) - normalizer = r"(\d+\.*)+" - - matched_strings = [max(x, key=len) for x in re.findall(full_regex_string, cert_name, re.IGNORECASE)] - if not matched_strings: - matched_strings = [max(x, key=len) for x in re.findall(at_least_something, cert_name, re.IGNORECASE)] - # Only keep the first occurrence but keep order. - matches = [] - for match in matched_strings: - if match not in matches: - matches.append(match) - # identified_versions = list(set([max(x, key=len) for x in re.findall(VERSION_PATTERN, cert_name, re.IGNORECASE | re.VERBOSE)])) - # return identified_versions if identified_versions else ['-'] - - if not matches: - return {constants.CPE_VERSION_NA} - - matched = [re.search(normalizer, x) for x in matches] - return {x.group() for x in matched if x is not None} - - -def tokenize_dataset(dset: list[str], keywords: set[str]) -> np.ndarray: - return np.array([tokenize(x, keywords) for x in dset]) - - -def tokenize(string: str, keywords: set[str]) -> str: - return " ".join([x for x in string.split() if x.lower() in keywords]) - - -def normalize_fips_vendor(string: str) -> str: - """ - "Normalizes" FIPS vendor. Precisely: - - Removes some punctuation and non-alphanumerical symbols - - Returns only first 5 tokens - # TODO: The rationale of the steps outlined above should be investigatated - """ - return " ".join( - string.replace("(R)", "").replace(",", "").replace("®", "").replace("-", " ").replace("+", " ").split()[:4] - ) - - -# Credit: https://stackoverflow.com/questions/18092354/ -def split_unescape(s: str, delim: str, escape: str = "\\", unescape: bool = True) -> list[str]: - """ - >>> split_unescape('foo,bar', ',') - ['foo', 'bar'] - >>> split_unescape('foo$,bar', ',', '$') - ['foo,bar'] - >>> split_unescape('foo$$,bar', ',', '$', unescape=True) - ['foo$', 'bar'] - >>> split_unescape('foo$$,bar', ',', '$', unescape=False) - ['foo$$', 'bar'] - >>> split_unescape('foo$', ',', '$', unescape=True) - ['foo$'] - """ - ret = [] - current = [] - itr = iter(s) - for ch in itr: - if ch == escape: - try: - # skip the next character; it has been escaped! - if not unescape: - current.append(escape) - current.append(next(itr)) - except StopIteration: - if unescape: - current.append(escape) - elif ch == delim: - # split! (add current to the list and reset it) - ret.append("".join(current)) - current = [] - else: - current.append(ch) - ret.append("".join(current)) - return ret - - -def warn_if_missing_poppler() -> None: - """ - Warns user if he misses a poppler dependency - """ - try: - if not pkgconfig.installed("poppler-cpp", ">=0.30"): - logger.warning( - "Attempting to run pipeline with pdf->txt conversion, but poppler-cpp dependency was not found." - ) - except OSError: - logger.warning("Attempting to find poppler-cpp, but pkg-config was not found.") - - -def warn_if_missing_tesseract() -> None: - """ - Warns user if he misses a tesseract dependency - """ - try: - if not pkgconfig.installed("tesseract", ">=5.0.0"): - logger.warning( - "Attempting to run pipeline with pdf->txt conversion, that requires tesseract, but tesseract was not found." - ) - except OSError: - logger.warning("Attempting to find tesseract, but pkg-config was not found.") - - -def choose_lowest_eal(eals: set[str] | None) -> str | None: - """ - Given a set of EAL strings, chooses the lowest one. - """ - if not eals: - return None - - matches = [(re.search(r"\d+", x)) for x in eals] - min_number = min([int(x.group()) for x in matches if x]) - candidates = [x for x in eals if str(min_number) in x] - return "EAL" + str(min_number) if len(candidates) == 2 else candidates[0] diff --git a/sec_certs/utils/pandas.py b/sec_certs/utils/pandas.py deleted file mode 100644 index 97068e77..00000000 --- a/sec_certs/utils/pandas.py +++ /dev/null @@ -1,542 +0,0 @@ -from __future__ import annotations - -import copy -import functools -import logging -import tempfile -import xml.etree.ElementTree as ET -import zipfile -from dataclasses import dataclass -from pathlib import Path -from shutil import copyfile -from typing import Any, Final - -import numpy as np -import pandas as pd -from matplotlib import pyplot as plt -from scipy import stats -from tqdm.notebook import tqdm - -from sec_certs.dataset.cve import CVEDataset -from sec_certs.sample.sar import SAR -from sec_certs.utils import helpers - -logger = logging.getLogger(__name__) - - -@dataclass(eq=True, frozen=True) -class SecondarySFPCluster: - name: str - children: frozenset[int] - - @classmethod - def from_xml_id(cls, xml_categories: list[ET.Element], cwe_id: int): - cat = cls.find_correct_category(xml_categories, cwe_id) - name = cat.attrib["Name"] - members = cat.find("{http://cwe.mitre.org/cwe-6}Relationships") - - assert members is not None - member_ids = frozenset( - int(x.attrib["CWE_ID"]) for x in members if x.tag == "{http://cwe.mitre.org/cwe-6}Has_Member" - ) - return cls(name, member_ids) - - @staticmethod - def find_correct_category(xml_categories: list[ET.Element], cwe_id: int) -> ET.Element: - for cat in xml_categories: - if cat.attrib["ID"] == str(cwe_id): - return cat - raise ValueError(f"Category with ID {cwe_id} found.") - - -@dataclass(eq=True, frozen=True) -class PrimarySFPCluster: - name: str - secondary_clusters: frozenset[SecondarySFPCluster] - cwe_ids: frozenset[int] - - @classmethod - def from_xml(cls, xml_categories: list[ET.Element], primary_cluster_element: ET.Element): - name = primary_cluster_element.attrib["Name"].split("SFP Primary Cluster: ")[1] - members = primary_cluster_element.find("{http://cwe.mitre.org/cwe-6}Relationships") - - assert members is not None - member_ids = {int(x.attrib["CWE_ID"]) for x in members if x.tag == "{http://cwe.mitre.org/cwe-6}Has_Member"} - - secondary_clusters = [] - cwe_ids = [] - for member_id in member_ids: - try: - secondary_clusters.append(SecondarySFPCluster.from_xml_id(xml_categories, member_id)) - except ValueError: - cwe_ids.append(member_id) - - return cls(name, frozenset(secondary_clusters), frozenset(cwe_ids)) - - -class SFPModel: - URL: Final[str] = "https://cwe.mitre.org/data/xml/views/888.xml.zip" - XML_FILENAME: Final[str] = "888.xml" - XML_ZIP_NAME: Final[str] = "888.xml.zip" - - def __init__(self, primary_clusters: frozenset[PrimarySFPCluster]): - self.primary_clusters = primary_clusters - - @classmethod - def from_xml(cls, xml_filepath: str | Path): - tree = ET.parse(xml_filepath) - category_tag = tree.getroot().find("{http://cwe.mitre.org/cwe-6}Categories") - - assert category_tag is not None - categories = category_tag.findall("{http://cwe.mitre.org/cwe-6}Category") - - # The XML contains two weird primary clusters not specified in https://samate.nist.gov/BF/Enlightenment/SFP.html. - # After manual inspection, we skip those - primary_clusters = frozenset( - PrimarySFPCluster.from_xml(categories, x) - for x in categories - if ( - "SFP Primary Cluster" in x.attrib["Name"] - and x.attrib["Name"] != "SFP Primary Cluster: Failure to Release Memory" - and x.attrib["Name"] != "SFP Primary Cluster: Faulty Resource Release" - ) - ) - - return cls(primary_clusters) - - @classmethod - def from_web(cls): - with tempfile.TemporaryDirectory() as tmp_dir: - xml_zip_path = Path(tmp_dir) / cls.XML_ZIP_NAME - helpers.download_file(cls.URL, xml_zip_path) - - with zipfile.ZipFile(xml_zip_path, "r") as zip_handle: - zip_handle.extractall(tmp_dir) - - return cls.from_xml(Path(tmp_dir) / cls.XML_FILENAME) - - def search_cwe(self, cwe_id: int) -> tuple[str | None, str | None]: - for primary in self.primary_clusters: - for secondary in primary.secondary_clusters: - if cwe_id in secondary.children: - return primary.name, secondary.name - if cwe_id in primary.cwe_ids: - return primary.name, None - return None, None - - -def discover_sar_families(ser: pd.Series) -> list[str]: - """ - Returns a list of all SAR families that occur in the pandas Series, where each entry is a set of SAR objects. - """ - sars = ser.tolist() - families = set() - for cert in sars: - families |= {x.family for x in cert} if not pd.isnull(cert) else set() - return list(families) - - -def get_sar_level_from_set(sars: set[SAR], sar_family: str) -> int | None: - """ - Given a set of SARs and a family name, will return level of the seeked SAR from the set. - """ - family_sars_dict = {x.family: x for x in sars} if (sars and not pd.isnull(sars)) else dict() - if sar_family not in family_sars_dict.keys(): - return None - return family_sars_dict[sar_family].level - - -def compute_cve_correlations( - df: pd.DataFrame, - exclude_vuln_free_certs: bool = False, - sar_families: list[str] | None = None, - output_path: str | Path | None = None, - filter_nans: bool = True, -) -> pd.DataFrame: - """ - Computes correlations of EAL and different SARs and two columns: (n_cves, worst_cve_score, avg_cve_score). Few assumptions about the passed dataframe: - - EAL column must be categorical data type - - SAR column must be a set of SARs - - `n_cves` and `worst_cve_score`, `avg_cve_score` columns must be present in the dataframe - Possibly, it can filter columns will both values NaN (due to division by zero or super low supports.) - To choose correct minimal support is tricky, this is because SAR levels often having huge support, but being imbalanced themselves heavily in the favor - of a single value that is rarely modified. We recommend choosing 100 and discarding any row where some column would result into NaN - """ - df_sar = df.loc[:, ["eal", "extracted_sars", "worst_cve_score", "avg_cve_score", "n_cves", "category"]] - df_sar = df_sar.loc[df_sar.category != "ICs, Smart Cards and Smart Card-Related Devices and Systems"] - - if exclude_vuln_free_certs: - df_sar = df_sar.loc[df_sar.n_cves > 0] - - families = sar_families if sar_families else discover_sar_families(df_sar.extracted_sars) - - spearmanr = functools.partial(stats.spearmanr, nan_policy="omit", alternative="less") - - df_sar.eal = df_sar.eal.cat.codes - df_sar.eal = df_sar.eal.map(lambda x: np.NaN if x == -1 else x) - - n_cves_eal_corr, n_cves_eal_pvalue = spearmanr(df_sar.eal, df_sar.n_cves) - n_cves_corrs = [n_cves_eal_corr] - n_cves_pvalues = [n_cves_eal_pvalue] - - worst_cve_eal_corr, worst_cve_eal_pvalue = spearmanr(df_sar.eal, df_sar.worst_cve_score) - worst_cve_corrs = [worst_cve_eal_corr] - worst_cve_pvalues = [worst_cve_eal_pvalue] - - avg_cve_eal_corr, avg_cve_eal_pvalue = spearmanr(df_sar.eal, df_sar.avg_cve_score) - avg_cve_corrs = [avg_cve_eal_corr] - avg_cve_pvalues = [avg_cve_eal_pvalue] - - supports = [df_sar.loc[~df_sar["eal"].isnull()].shape[0]] - - for family in tqdm(families): - df_sar[family] = df_sar.extracted_sars.map(lambda x: get_sar_level_from_set(x, family)) - - n_cves_corr, n_cves_pvalue = spearmanr(df_sar[family], df_sar.n_cves) - n_cves_corrs.append(n_cves_corr) - n_cves_pvalues.append(n_cves_pvalue) - - worst_cve_corr, worst_cve_pvalue = spearmanr(df_sar[family], df_sar.worst_cve_score) - worst_cve_corrs.append(worst_cve_corr) - worst_cve_pvalues.append(worst_cve_pvalue) - - avg_cve_corr, avg_cve_pvalue = spearmanr(df_sar[family], df_sar.avg_cve_score) - avg_cve_corrs.append(avg_cve_corr) - avg_cve_pvalues.append(avg_cve_pvalue) - - supports.append(df_sar.loc[~df_sar[family].isnull()].shape[0]) - - df_sar = df_sar.copy() - - tuples = list( - zip(n_cves_corrs, n_cves_pvalues, worst_cve_corrs, worst_cve_pvalues, avg_cve_corrs, avg_cve_pvalues, supports) - ) - dct = {family: correlations for family, correlations in zip(["eal"] + families, tuples)} - df_corr = pd.DataFrame.from_dict( - dct, - orient="index", - columns=[ - "n_cves_corr", - "n_cves_pvalue", - "worst_cve_score_corr", - "worst_cve_pvalue", - "avg_cve_score_corr", - "avg_cve_pvalue", - "support", - ], - ) - df_corr.style.set_caption("Correlations between EAL, SARs and CVEs") - df_corr = df_corr.sort_values(by="support", ascending=False) - - if filter_nans: - df_corr = df_corr.dropna(how="any", subset=["n_cves_corr", "worst_cve_score_corr", "avg_cve_score_corr"]) - - if output_path: - df_corr.to_csv(output_path) - - return df_corr - - -def find_earliest_maintenance_after_cve(row): - "Given dataframe row, will return first maintenance date succeeding first published CVE related to a certificate if exists, else np.nan" - maintenances_after_cve = [x for x in row["maintenance_dates"] if x > row["earliest_cve"]] - return min(maintenances_after_cve) if maintenances_after_cve else np.nan - - -def filter_to_cves_within_validity_period(cc_df: pd.DataFrame, cve_dset: CVEDataset) -> pd.DataFrame: - """ - Filters the column `related_cves` in `cc_df` DataFrame to CVEs that were published within validity period of the - studied certificate. - """ - - def filter_cves( - cve_dset: CVEDataset, cves: set[str], not_valid_before: pd.Timestamp, not_valid_after: pd.Timestamp - ) -> set[str] | float: - - # Mypy is complaining, but the Optional date is resolved at the beginning of the and condition - result: set[str] = { - x - for x in cves - if cve_dset[x].published_date - and not_valid_before < pd.Timestamp(cve_dset[x].published_date.date()) # type: ignore - and not_valid_after > pd.Timestamp(cve_dset[x].published_date.date()) # type: ignore - } - - return result if result else np.nan - - if ( - cc_df.loc[ - (cc_df.related_cves.notnull()) & ((cc_df.not_valid_before.isna()) | (cc_df.not_valid_after.isna())) - ].shape[0] - > 0 - ): - raise ValueError( - "Cannot filter CVEs on certificates that have NaNs in not_valid_after or not_valid_before fields." - ) - - cc_df["related_cves"] = cc_df.apply( - lambda row: filter_cves(cve_dset, row["related_cves"], row["not_valid_before"], row["not_valid_after"]) - if not pd.isna(row["related_cves"]) - else row["related_cves"], - axis=1, - ) - - return cc_df - - -def expand_df_with_cve_cols(df: pd.DataFrame, cve_dset: CVEDataset) -> pd.DataFrame: - df = df.copy() - - df["n_cves"] = df.related_cves.map(lambda x: len(x) if x is not np.nan else 0) - df["cve_published_dates"] = df.related_cves.map( - lambda x: [cve_dset[y].published_date.date() for y in x] if x is not np.nan else np.nan # type: ignore - ) - - df["earliest_cve"] = df.cve_published_dates.map(lambda x: min(x) if isinstance(x, list) else np.nan) - df["worst_cve_score"] = df.related_cves.map( - lambda x: max([cve_dset[cve].impact.base_score for cve in x]) if x is not np.nan else np.nan - ) - - """ - Note: Technically, CVE can have 0 base score. This happens when the CVE is discarded from the database. - This could skew the results. During May 2022 analysis, we encountered a single CVE with such score. - Therefore, we do not treat this case. - To properly treat this, the average should be taken across CVEs with >0 base_socre. - """ - df["avg_cve_score"] = df.related_cves.map( - lambda x: np.mean([cve_dset[cve].impact.base_score for cve in x]) if x is not np.nan else np.nan - ) - return df - - -def prepare_cwe_df( - cc_df: pd.DataFrame, cve_dset: CVEDataset, fine_grained: bool = False -) -> tuple[pd.DataFrame, pd.DataFrame]: - """ - This function does the following: - 1. Filter CC DF to columns relevant for CWE examination (eal, related_cves, category) - 2. Parses CWE webpage of CWE categories and weaknesses, fetches CWE descriptions and names from there - 3. Explodes the CC DF so that each row corresponds to single CVE - 4. Joins CC DF with CWE DF obtained from CVEDataset - 5. Explodes resulting DF again so that each row corresponds to single CWE - - :param pd.DataFrame cc_df: DataFrame obtained from CCDataset, should be limited to rows with >0 vulnerabilities - :param CVEDataset cve_dset: CVEDataset instance to retrieve CWE data from - :param bool fine_grained: If se to True, CWEs won't be merged into weaknesses of higher abstraction - :return Tuple[pd.DataFrame, pd.DataFrame]: returns two dataframes: - - DF obtained from CC Dataset, fully exploded to CWEs - - DF obtained from CWE webpage, contains IDs, names, types, urls of all CWEs - """ - # Explode CVE_IDs and CWE_IDs so that we have right counts on duplicated CVEs. Measure how much data for analysis we have left. - vulns = cve_dset.to_pandas() - df_cwe_relevant = ( - cc_df[["eal", "related_cves", "category"]] - .explode(column="related_cves") - .rename(columns={"related_cves": "cve_id"}) - ) - df_cwe_relevant["cwe_ids"] = df_cwe_relevant.cve_id.map(lambda x: vulns.cwe_ids[x]) - df_cwe_relevant = ( - df_cwe_relevant.explode(column="cwe_ids") - .reset_index() - .rename(columns={"cwe_ids": "cwe_id", "index": "cert_dgst"}) - ) - - df_cwe_relevant.cwe_id = df_cwe_relevant.cwe_id.replace(r"NVD-CWE-*", np.nan, regex=True) - print( - f"Filtering {df_cwe_relevant.loc[df_cwe_relevant.cwe_id.isna(), 'cve_id'].nunique()} CVEs that have no CWE assigned. This affects {df_cwe_relevant.loc[df_cwe_relevant.cwe_id.isna(), 'cert_dgst'].nunique()} certificates" - ) - print( - f"Still left with analysis of {df_cwe_relevant.loc[~df_cwe_relevant.cwe_id.isna(), 'cve_id'].nunique()} CVEs in {df_cwe_relevant.loc[~df_cwe_relevant.cwe_id.isna(), 'cert_dgst'].nunique()} certificates." - ) - df_cwe_relevant = df_cwe_relevant.dropna() - - # Load CWE IDs and descriptions from CWE website - with tempfile.TemporaryDirectory() as tmp_dir: - xml_zip_path = Path(tmp_dir) / "cwec_latest.xml.zip" - helpers.download_file("https://cwe.mitre.org/data/xml/cwec_latest.xml.zip", xml_zip_path) - - with zipfile.ZipFile(xml_zip_path, "r") as zip_handle: - zip_handle.extractall(tmp_dir) - xml_filename = zip_handle.namelist()[0] - - root = ET.parse(Path(tmp_dir) / xml_filename).getroot() - - weaknesses = root.find("{http://cwe.mitre.org/cwe-6}Weaknesses") - categories = root.find("{http://cwe.mitre.org/cwe-6}Categories") - dct: dict[str, Any] = { - "cwe_id": [], - "cwe_name": [], - "cwe_description": [], - "type": [], - "child_of": [], - } - - assert weaknesses - for weakness in weaknesses: - assert weakness - description = weakness.find("{http://cwe.mitre.org/cwe-6}Description") - related_weaknesses = weakness.find("{http://cwe.mitre.org/cwe-6}Related_Weaknesses") - - dct["cwe_id"].append("CWE-" + weakness.attrib["ID"]) - dct["cwe_name"].append(weakness.attrib["Name"]) - dct["cwe_description"].append(description.text if description is not None else None) - dct["type"].append("weakness") - - if related_weaknesses: - dct["child_of"].append( - { - "CWE-" + x.attrib["CWE_ID"] - for x in related_weaknesses - if x.tag == "{http://cwe.mitre.org/cwe-6}Related_Weakness" and x.attrib["Nature"] == "ChildOf" - } - ) - else: - dct["child_of"].append(np.nan) - - assert categories - for category in categories: - assert category - summary = category.find("{http://cwe.mitre.org/cwe-6}Summary") - - dct["cwe_id"].append("CWE-" + category.attrib["ID"]) - dct["cwe_name"].append(category.attrib["Name"]) - dct["cwe_description"].append(summary.text if summary is not None else None) - dct["type"].append("category") - dct["child_of"].append(np.nan) - - cwe_df = pd.DataFrame(dct).set_index("cwe_id") - cwe_df["url"] = cwe_df.index.map(lambda x: "https://cwe.mitre.org/data/definitions/" + x.split("-")[1] + ".html") - cwe_df = cwe_df.replace(r"\n", " ", regex=True) - - if fine_grained: - return df_cwe_relevant, cwe_df - else: - return get_coarse_grained_cwes(df_cwe_relevant, cwe_df), cwe_df - - -def get_coarse_grained_cwes(fine_grained_df: pd.DataFrame, cwe_df: pd.DataFrame) -> pd.DataFrame: - """ - Oddly enough, NVD contains CWEs at different levels of abstraction, which makes it difficult to compare between them. - Among others, some three different CWEs appear in the CVEDataset: CWE-20, CWE-119, CWE-787. Problem is that CWE-787 - is child of CWE-119, which in turn is child of CWE-20. It makes no sense to compute stats of most prevalent CWEs - unless categories are aligned to the top-most level. - - This function aligns the categories to the top-most level. It works in loop. When an iteration is performed without - replacing any CWEs with their parents, the algorithm terminates. - The algorithm inspects every CWE and replaces it with all its parents on condition that they appear in the CVE Dataset. - - :param pd.DataFrame fine_grained_df: First element of the output of `prepare_cwe_df` function - :param pd.DataFrame cwe_df: Second element of the output of `prepare_cwe_df` function - :return pd.DataFrame: DF obtained from CC Dataset, fully exploded to coarse-grained CWEs - """ - all_cwes_in_original_df = set(fine_grained_df.cwe_id.unique()) - parent_dict = cwe_df.child_of.to_dict() - new_set = set(fine_grained_df.cwe_id.unique()) - mapping = {x: {x} for x in new_set} - - while True: - old_set = copy.deepcopy(new_set) - for cwe in old_set: - parents = parent_dict[cwe] - if parents and parents is not np.nan and any(x in all_cwes_in_original_df for x in parents): - new_set.remove(cwe) - new_set.update({x for x in parents if x in all_cwes_in_original_df}) - for val in mapping.values(): - if cwe in val: - val.remove(cwe) - val.update({x for x in parents if x in all_cwes_in_original_df}) - if new_set == old_set: - break - - # Now we should have complete mapping of fine_grained -> coarse_grained CWEs - new_df = fine_grained_df.copy() - new_df.cwe_id = new_df.cwe_id.map(mapping) - - return new_df.explode(column="cwe_id") - - -def get_top_n_cwes( - df: pd.DataFrame, cwe_df: pd.DataFrame, category: str | None = None, eal: str | None = None, n_cwes: int = 10 -) -> pd.DataFrame: - """Fetches top-n CWEs, overall, per category, or per EAL""" - top_n = df.copy() - - if category: - top_n = top_n.loc[top_n.category == category].copy() - if eal: - top_n = top_n.loc[top_n.eal == eal].copy() - - top_n = ( - top_n.cwe_id.value_counts() - .head(n_cwes) - .to_frame() - .rename(columns={"cwe_id": "frequency"}) - .rename_axis("cwe_id") - ) - top_n["cwe_name"] = top_n.index.map(lambda x: cwe_df.loc[x].cwe_name) - top_n["cwe_description"] = top_n.index.map(lambda x: cwe_df.loc[x].cwe_description) - top_n["url"] = top_n.index.map(lambda x: cwe_df.loc[x].url) - top_n["type"] = top_n.index.map(lambda x: cwe_df.loc[x].type) - - return top_n - - -def compute_maintenances_that_come_after_vulns(df: pd.DataFrame) -> pd.DataFrame: - """ - Given pre-processed CCDataset DataFrame (expanded with MU & CVE cols), computes time to fix CVE and earliest CVE after some vuln. - """ - df_fixed = df.loc[(df.n_cves > 0) & (df.n_maintenances > 0)].copy() - df_fixed.maintenance_dates = df_fixed.maintenance_dates.map(lambda x: [y.date() for y in x]) - df_fixed.loc[:, "earliest_maintenance_after_vuln"] = df_fixed.apply(find_earliest_maintenance_after_cve, axis=1) - df_fixed.index.name = "dgst" - return df_fixed - - -def move_fixing_mu_to_directory( - df_fixed: pd.DataFrame, main_df: pd.DataFrame, outdir: str | Path, inpath: str | Path -) -> pd.DataFrame: - """ - Localizes reports of maintenance updates that should fix some vulnerability and copies them into a directory. - df_fixed should be the output of compute_maintenances_that_come_after_vulns method. - """ - fixed_df_index = ( - df_fixed.loc[~df_fixed.earliest_maintenance_after_vuln.isnull()] - .reset_index() - .set_index(["dgst", "earliest_maintenance_after_vuln"]) - .index.to_flat_index() - ) - main_df.maintenance_date = main_df.maintenance_date.map(lambda x: x.date()) - main_prefiltered = main_df.reset_index().set_index(["related_cert_digest", "maintenance_date"]) - mu_filenames = main_prefiltered.loc[main_prefiltered.index.isin(fixed_df_index), "dgst"] - mu_filenames = mu_filenames.map(lambda x: x + ".pdf") - - inpath = Path(inpath) - if not inpath.exists(): - inpath.mkdir() - - for i in mu_filenames: - copyfile(inpath / i, Path(outdir) / i) - - return mu_filenames - - -def plot_dataframe_graph( - data: dict, - label: str, - file_name: str, - density: bool = False, - cumulative: bool = False, - bins: int = 50, - log: bool = True, - show: bool = True, -) -> None: - pd_data = pd.Series(data) - pd_data.hist(bins=bins, label=label, density=density, cumulative=cumulative) - plt.savefig(file_name) - if show: - plt.show() - - if log: - sorted_data = pd_data.value_counts(ascending=True) - - logger.info(sorted_data.where(sorted_data > 1).dropna()) diff --git a/sec_certs/utils/parallel_processing.py b/sec_certs/utils/parallel_processing.py deleted file mode 100644 index 50806a67..00000000 --- a/sec_certs/utils/parallel_processing.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -import time -from multiprocessing.pool import ThreadPool -from typing import Any, Callable, Iterable - -from billiard.pool import Pool - -from sec_certs.utils.tqdm import tqdm - - -def process_parallel( - func: Callable, - items: Iterable, - max_workers: int, - callback: Callable | None = None, - use_threading: bool = True, - progress_bar: bool = True, - unpack: bool = False, - progress_bar_desc: str | None = None, -) -> list[Any]: - - pool: Pool | ThreadPool = ThreadPool(max_workers) if use_threading else Pool(max_workers) - results = ( - [pool.apply_async(func, (*i,), callback=callback) for i in items] - if unpack - else [pool.apply_async(func, (i,), callback=callback) for i in items] - ) - - if progress_bar is True and items: - bar = tqdm(total=len(results), desc=progress_bar_desc) - while not all(all_done := [x.ready() for x in results]): - done_count = len(list(filter(lambda x: x, all_done))) - bar.update(done_count - bar.n) - time.sleep(1) - bar.update(len(results) - bar.n) - bar.close() - - pool.close() - pool.join() - pool.terminate() - - return [r.get() for r in results] diff --git a/sec_certs/utils/pdf.py b/sec_certs/utils/pdf.py deleted file mode 100644 index 1d04a697..00000000 --- a/sec_certs/utils/pdf.py +++ /dev/null @@ -1,275 +0,0 @@ -from __future__ import annotations - -import glob -import logging -import subprocess -from datetime import datetime, timedelta, timezone -from functools import reduce -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Any - -import pdftotext -import pikepdf -from PyPDF2 import PdfFileReader -from PyPDF2.generic import BooleanObject, ByteStringObject, FloatObject, IndirectObject, NumberObject, TextStringObject - -from sec_certs import constants as constants -from sec_certs.constants import ( - GARBAGE_ALPHA_CHARS_THRESHOLD, - GARBAGE_AVG_LLEN_THRESHOLD, - GARBAGE_EVERY_SECOND_CHAR_THRESHOLD, - GARBAGE_LINES_THRESHOLD, - GARBAGE_SIZE_THRESHOLD, -) - -logger = logging.getLogger(__name__) - - -def repair_pdf(file: Path) -> None: - """ - Some pdfs can't be opened by PyPDF2 - opening them with pikepdf and then saving them fixes this issue. - By opening this file in a pdf reader, we can already extract number of pages. - - :param file: file name - :return: number of pages in pdf file - """ - pdf = pikepdf.Pdf.open(file, allow_overwriting_input=True) - pdf.save(file) - - -def ocr_pdf_file(pdf_path: Path) -> str: - """ - OCR a PDF file and return its text contents, uses `pdftoppm` and `tesseract`. - - :param pdf_path: The PDF file to OCR. - :return: The text contents. - """ - with TemporaryDirectory() as tmpdir: - tmppath = Path(tmpdir) - ppm = subprocess.run( - ["pdftoppm", pdf_path, tmppath / "image"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - if ppm.returncode != 0: - raise ValueError(f"pdftoppm failed: {ppm.returncode}") - for ppm_path in map(Path, glob.glob(str(tmppath / "image*.ppm"))): - base = ppm_path.with_suffix("") - tes = subprocess.run( - ["tesseract", "-l", "eng+deu+fra", ppm_path, base], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - if tes.returncode != 0: - raise ValueError(f"tesseract failed: {tes.returncode}") - contents = "" - txt_paths = list(glob.glob(str(tmppath / "image*.txt"))) - for txt_path in map(Path, sorted(txt_paths, key=lambda fname: int(fname[6:-4]))): - with txt_path.open("r", encoding="utf-8") as f: - contents += f.read() - return contents - - -def convert_pdf_file(pdf_path: Path, txt_path: Path) -> tuple[bool, bool]: - """ - Convert a PDF tile to text and save it on the `txt_path`. - - :param pdf_path: Path to the to-be-converted PDF file. - :param txt_path: Path to the resulting text file. - :return: A tuple of two results, whether OCR was done and what the complete result - was (OK/NOK). - """ - txt = None - ok = False - ocr = False - try: - with pdf_path.open("rb") as pdf_handle: - pdf = pdftotext.PDF(pdf_handle, "", True) # No password, Raw=True - txt = "".join(pdf) - except Exception as e: - logger.error(f"Error when converting pdf->txt: {e}") - - if txt is None or text_is_garbage(txt): - logger.warning(f"Detected garbage during conversion of {pdf_path}") - ocr = True - try: - txt = ocr_pdf_file(pdf_path) - logger.info(f"OCR OK for {pdf_path}") - except Exception as e: - logger.error(f"Error during OCR of {pdf_path}, using garbage: {e}") - - if txt is not None: - ok = True - with txt_path.open("w", encoding="utf-8") as txt_handle: - txt_handle.write(txt) - - return ocr, ok - - -def parse_pdf_date(dateval: bytes | None) -> datetime | None: - """ - Parse PDF metadata date format: - - ``` - parse_pdf_date(b"D:20110617082321-04'00'") - ``` - into - ``` - datetime.datetime(2011, 6, 17, 8, 23, 21, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=72000))) - ``` - - :param dateval: The date as in the PDF metadata. - :return: The parsed datetime, if successful, else `None`. - """ - if dateval is None: - return None - clean = dateval.decode("utf-8").replace("D:", "") - tz = None - tzoff = None - if "+" in clean: - clean, tz = clean.split("+") - tzoff = 1 - if "-" in clean: - clean, tz = clean.split("-") - tzoff = -1 - elif "Z" in clean: - clean, tz = clean.split("Z") - tzoff = 1 - try: - res_datetime = datetime.strptime(clean, "%Y%m%d%H%M%S") - if tz and tzoff: - tz_datetime = datetime.strptime(tz, "%H'%M'") - delta = tzoff * timedelta(hours=tz_datetime.hour, minutes=tz_datetime.minute) - res_tz = timezone(delta) - res_datetime = res_datetime.replace(tzinfo=res_tz) - return res_datetime - except ValueError: - return None - - -def extract_pdf_metadata(filepath: Path) -> tuple[str, dict[str, Any] | None]: # noqa: C901 - """ - Extract PDF metadata, such as the number of pages, author, title, etc. - - :param filepath: THe path to the PDF. - :return: A tuple of the result code (see constants) and the metadata dictionary. - """ - - def map_metadata_value(val, nope_out=False): - if isinstance(val, BooleanObject): - val = val.value - elif isinstance(val, FloatObject): - val = float(val) - elif isinstance(val, NumberObject): - val = int(val) - elif isinstance(val, IndirectObject) and not nope_out: - # Let's make sure to nope out in case of cycles - val = map_metadata_value(val.getObject(), nope_out=True) - elif isinstance(val, TextStringObject): - val = str(val) - elif isinstance(val, ByteStringObject): - try: - val = val.decode("utf-8") - except UnicodeDecodeError: - val = str(val) - else: - val = str(val) - return val - - def resolve_indirect(val, bound=10): - if isinstance(val, list) and bound: - return [resolve_indirect(v, bound - 1) for v in val] - elif isinstance(val, IndirectObject) and bound: - return resolve_indirect(val.getObject(), bound - 1) - else: - return val - - metadata: dict[str, Any] = dict() - - try: - metadata["pdf_file_size_bytes"] = filepath.stat().st_size - with filepath.open("rb") as handle: - pdf = PdfFileReader(handle, strict=False) - metadata["pdf_is_encrypted"] = pdf.getIsEncrypted() - - # see https://stackoverflow.com/questions/26242952/pypdf-2-decrypt-not-working - if metadata["pdf_is_encrypted"]: - pikepdf.open(filepath, allow_overwriting_input=True).save() - - with filepath.open("rb") as handle: - pdf = PdfFileReader(handle, strict=False) - metadata["pdf_number_of_pages"] = pdf.getNumPages() - pdf_document_info = pdf.getDocumentInfo() - - if pdf_document_info is None: - raise ValueError("PDF metadata unavailable") - - for key, val in pdf_document_info.items(): - metadata[str(key)] = map_metadata_value(val) - - # Get the hyperlinks in the PDF - annots = [page.get("/Annots", []) for page in pdf.pages] - annots = reduce(lambda x, y: x + y, map(resolve_indirect, annots)) - links = set() - for annot in annots: - try: - A = resolve_indirect(annot.get("/A", {})) - link = resolve_indirect(A.get("/URI")) - if link: - links.add(map_metadata_value(link)) - except Exception: - pass - metadata["pdf_hyperlinks"] = links - - except Exception as e: - relative_filepath = "/".join(str(filepath).split("/")[-4:]) - error_msg = f"Failed to read metadata of {relative_filepath}, error: {e}" - logger.error(error_msg) - return error_msg, None - - return constants.RETURNCODE_OK, metadata - - -def text_is_garbage(text: str) -> bool: - """ - Detect whether the given text is "garbage". A series of tests is applied, - using the number of lines, average line length, total size, every second character on a line - and the ratio of alphanumeric characters. - - :param text: The tested text. - :return: Whether the text is a "garbage" result of pdftotext conversion. - """ - size = len(text) - content_len = 0 - lines = 0 - every_second = 0 - alpha_len = len("".join(filter(str.isalpha, text))) - for line in text.splitlines(): - content_len += len(line) - lines += 1 - if len(set(line[1::2])) > 1: - every_second += 1 - - if lines: - avg_line_len = content_len / lines - else: - avg_line_len = 0 - if size: - alpha = alpha_len / size - else: - alpha = 0 - - # If number of lines is small, this is garbage. - if lines < GARBAGE_LINES_THRESHOLD: - return True - # If the file size is small, this is garbage. - if size < GARBAGE_SIZE_THRESHOLD: - return True - # If the average length of a line is small, this is garbage. - if avg_line_len < GARBAGE_AVG_LLEN_THRESHOLD: - return True - # If there a small amount of lines that have more than one character at every second character, this is garbage. - # This detects the ANSSI spacing issues. - if every_second < GARBAGE_EVERY_SECOND_CHAR_THRESHOLD: - return True - # If there is a small ratio of alphanumeric chars to all chars, this is garbage. - if alpha < GARBAGE_ALPHA_CHARS_THRESHOLD: - return True - return False diff --git a/sec_certs/utils/sanitization.py b/sec_certs/utils/sanitization.py deleted file mode 100644 index 2f9cd046..00000000 --- a/sec_certs/utils/sanitization.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -import html -import logging -from datetime import date - -import numpy as np -import pandas as pd -from bs4 import NavigableString - -logger = logging.getLogger(__name__) - - -def sanitize_navigable_string(string: NavigableString | str | None) -> str | None: - if not string: - return None - return str(string).strip().replace("\xad", "").replace("\xa0", "") - - -def sanitize_link(record: str | None) -> str | None: - if not record: - return None - return record.replace(":443", "").replace(" ", "%20").replace("http://", "https://") - - -def sanitize_date(record: pd.Timestamp | date | np.datetime64) -> date | None: - if pd.isnull(record): - return None - elif isinstance(record, pd.Timestamp): - return record.date() - elif isinstance(record, (date, type(None))): - return record - raise ValueError("Unsupported type given as input") - - -def sanitize_string(record: str) -> str: - # There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I &#38; J' that has to be unescaped twice - string = html.unescape(html.unescape(record)).replace("\n", "") - return " ".join(string.split()) - - -def sanitize_security_levels(record: str | set[str]) -> set[str]: - if isinstance(record, str): - record = set(record.split(",")) - return record - {"Basic", "ND-PP", "PP\xa0Compliant", "None", "Medium"} - - -def sanitize_protection_profiles(record: str) -> list: - if not record: - return [] - return record.split(",") diff --git a/sec_certs/utils/tables.py b/sec_certs/utils/tables.py deleted file mode 100644 index 29def971..00000000 --- a/sec_certs/utils/tables.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -import logging -import re -from pathlib import Path - -from sec_certs.cert_rules import FIPS_LIST_OF_TABLES - -logger = logging.getLogger(__name__) - - -def parse_list_of_tables(txt: str) -> set[int]: - """ - Parses list of tables in policy txt, returns page numbers of tables that mention algorithms - """ - rr = re.compile(r"^.+?(?:[Ff]unction|[Aa]lgorithm|[Ss]ecurity [Ff]unctions?).+?(?P\d+)$", re.MULTILINE) - return {int(m.group("page_num")) for m in rr.finditer(txt)} - - -def get_table_rich_page_numbers_from_footer(file_text: str) -> set[int]: - """ - Parses page numbers of policy txt pages that may contain tables with algorithm data - """ - current_page = 1 - pages = set() - - for line in file_text.split("\n"): - if "\f" in line: - current_page += 1 - if line.startswith("Table ") or line.startswith("Exhibit"): - pages.add(current_page) - pages.add(current_page + 1) - if current_page > 2: - pages.add(current_page - 1) - - for page in pages: - if page > current_page - 1: - return pages - {page} - - return pages - - -def find_pages_with_tables(txt_filepath: Path) -> set[int]: - """ - Identifies pages in txt file that may contain tables. Return their page numbers. - """ - with txt_filepath.open("r", encoding="utf-8") as handle: - txt = handle.read() - - # Parse page numbers from list of tables if available - # Else look for "Table" in text and \f representing footer, then extract page number from footer - if list_of_tables := FIPS_LIST_OF_TABLES.search(txt): - result = parse_list_of_tables(list_of_tables.group()) - else: - result = get_table_rich_page_numbers_from_footer(txt) - - return result if result else set() - - -def get_algs_from_table(dataframe_text: str) -> set[str]: - reg = r"(?:#?\s?|(?:Cert)\.?[^. ]*?\s?)(?:[CcAa]\s)?(?P[CcAa]? ?\d+)" - return {m.group() for m in re.finditer(reg, dataframe_text)} diff --git a/sec_certs/utils/tqdm.py b/sec_certs/utils/tqdm.py deleted file mode 100644 index 77eeae94..00000000 --- a/sec_certs/utils/tqdm.py +++ /dev/null @@ -1,9 +0,0 @@ -from tqdm import tqdm as tqdm_original - -from sec_certs.config.configuration import config - - -def tqdm(*args, **kwargs): - if "disable" in kwargs: - return tqdm_original(*args, **kwargs) - return tqdm_original(*args, **kwargs, disable=not config.enable_progress_bars) diff --git a/setup.py b/setup.py deleted file mode 100644 index 1db53172..00000000 --- a/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -from setuptools import find_packages, setup - -setup( - name="sec-certs", - author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky, Jiri Michalik", - author_email="svenda@fi.muni.cz", - packages=find_packages(), - license="MIT", - description="Tool for analysis of security certificates", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - classifiers=[ - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: MIT License", - "Topic :: Security", - "Topic :: Security :: Cryptography", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - ], - python_requires=">=3.8", - install_requires=open("requirements/requirements.in").read().splitlines(), - extras_require={ - "dev": open("requirements/dev_requirements.in").read().splitlines(), - "test": open("requirements/test_requirements.in").read().splitlines(), - }, - include_package_data=True, - package_data={"sec_certs": ["settings.yaml", "settings-schema.json", "rules.yaml"]}, - entry_points={"console_scripts": ["sec-certs=sec_certs.cli:main"]}, - project_urls={ - "Project homepage": "https://seccerts.org", - "GitHub repository": "https://github.com/crocs-muni/sec-certs/", - "Documentation": "https://seccerts.org/docs", - }, -) diff --git a/src/sec_certs/__init__.py b/src/sec_certs/__init__.py new file mode 100644 index 00000000..816ea2b8 --- /dev/null +++ b/src/sec_certs/__init__.py @@ -0,0 +1,7 @@ +""" +Tool for analysis of security certificates and their security targets (Common Criteria, NIST FIPS140-2...). +Contains three main sub-packages: +- dataset - package that holds the respective datasets and performs all processing of them +- sample - package that holds a single sample (e.g., Common Criteria certificate - CommonCriteriaCert). Mostly data structure, but can provide basic functionality. +- model - package that provides data pipelines (transformers, classifiers, ...) for complex transformations of datasets. +""" diff --git a/src/sec_certs/__main__.py b/src/sec_certs/__main__.py new file mode 100644 index 00000000..1d699b85 --- /dev/null +++ b/src/sec_certs/__main__.py @@ -0,0 +1,6 @@ +import sys + +from sec_certs.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sec_certs/cert_rules.py b/src/sec_certs/cert_rules.py new file mode 100644 index 00000000..145566f7 --- /dev/null +++ b/src/sec_certs/cert_rules.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Final + +import yaml + +# This ignores ACM and AMA SARs that are present in CC version 2 +SARS_IMPLIED_FROM_EAL: dict[str, set[tuple[str, int]]] = { + "EAL1": { + ("ADV_FSP", 1), + ("AGD_OPE", 1), + ("AGD_PRE", 1), + ("ALC_CMC", 1), + ("ALC_CMS", 1), + ("ASE_CCL", 1), + ("ASE_ECD", 1), + ("ASE_INT", 1), + ("ASE_OBJ", 1), + ("ASE_REQ", 1), + ("ASE_TSS", 1), + ("ATE_IND", 1), + ("AVA_VAN", 1), + }, + "EAL2": { + ("ADV_ARC", 1), + ("ADV_TDS", 1), + ("AGD_OPE", 1), + ("AGD_PRE", 1), + ("ALC_CMC", 2), + ("ALC_CMS", 2), + ("ALC_DEL", 1), + ("ASE_CCL", 1), + ("ASE_ECD", 1), + ("ASE_INT", 1), + ("ASE_OBJ", 2), + ("ASE_REQ", 2), + ("ASE_SPD", 1), + ("ASE_TSS", 1), + ("ATE_COV", 1), + ("ATE_FUN", 1), + ("ATE_IND", 2), + ("AVA_VAN", 2), + }, + "EAL3": { + ("ADV_ARC", 1), + ("ADV_FSP", 3), + ("ADV_TDS", 2), + ("AGD_PRE", 1), + ("ALC_CMC", 3), + ("ALC_CMS", 3), + ("ALC_DEL", 1), + ("ALC_DVS", 1), + ("ALC_LCD", 1), + ("ASE_CCL", 1), + ("ASE_ECD", 1), + ("ASE_INT", 1), + ("ASE_OBJ", 2), + ("ASE_REQ", 2), + ("ASE_SPD", 1), + ("ASE_TSS", 1), + ("ATE_COV", 2), + ("ATE_DPT", 1), + ("ATE_FUN", 1), + ("ATE_IND", 2), + ("AVA_VAN", 2), + }, + "EAL4": { + ("ADV_ARC", 1), + ("ADV_FSP", 4), + ("ADV_IMP", 1), + ("ADV_TDS", 3), + ("AGD_OPE", 1), + ("AGD_PRE", 1), + ("ALC_CMC", 4), + ("ALC_CMS", 4), + ("ALC_DEL", 1), + ("ALC_DVS", 1), + ("ALC_LCD", 1), + ("ALC_TAT", 1), + ("ASE_CCL", 1), + ("ASE_ECD", 1), + ("ASE_INT", 1), + ("ASE_OBJ", 2), + ("ASE_REQ", 2), + ("ASE_SPD", 1), + ("ASE_TSS", 1), + ("ATE_COV", 2), + ("ATE_DPT", 1), + ("ATE_FUN", 1), + ("ATE_IND", 2), + ("AVA_VAN", 3), + }, + "EAL5": { + ("ADV_ARC", 1), + ("ADV_FSP", 5), + ("ADV_IMP", 1), + ("ADV_INT", 2), + ("ADV_TDS", 4), + ("AGD_OPE", 1), + ("AGD_PRE", 1), + ("ALC_CMC", 4), + ("ALC_CMS", 5), + ("ALC_DEL", 1), + ("ALC_DVS", 1), + ("ALC_LCD", 1), + ("ALC_TAT", 2), + ("ASE_CCL", 1), + ("ASE_ECD", 1), + ("ASE_INT", 1), + ("ASE_OBJ", 2), + ("ASE_REQ", 2), + ("ASE_SPD", 1), + ("ASE_TSS", 1), + ("ATE_COV", 2), + ("ATE_DPT", 3), + ("ATE_FUN", 1), + ("ATE_IND", 2), + ("AVA_VAN", 4), + }, + "EAL6": { + ("ADV_ARC", 1), + ("ADV_FSP", 5), + ("ADV_IMP", 2), + ("ADV_INT", 3), + ("ADV_SPM", 1), + ("ADV_TDS", 5), + ("AGD_OPE", 1), + ("AGD_PRE", 1), + ("ALC_CMC", 5), + ("ALC_CMS", 5), + ("ALC_DEL", 1), + ("ALC_DVS", 2), + ("ALC_LCD", 1), + ("ALC_TAT", 3), + ("ASE_CCL", 1), + ("ASE_ECD", 1), + ("ASE_INT", 1), + ("ASE_OBJ", 2), + ("ASE_REQ", 2), + ("ASE_SPD", 1), + ("ASE_TSS", 1), + ("ATE_COV", 3), + ("ATE_DPT", 3), + ("ATE_FUN", 2), + ("ATE_IND", 2), + ("AVA_VAN", 5), + }, + "EAL7": { + ("ADV_ARC", 1), + ("ADV_FSP", 6), + ("ADV_IMP", 2), + ("ADV_INT", 3), + ("ADV_SPM", 1), + ("ADV_TDS", 6), + ("AGD_OPE", 1), + ("AGD_PRE", 1), + ("ALC_CMC", 5), + ("ALC_CMS", 5), + ("ALC_DEL", 1), + ("ALC_DVS", 2), + ("ALC_LCD", 2), + ("ALC_TAT", 3), + ("ASE_CCL", 1), + ("ASE_ECD", 1), + ("ASE_INT", 1), + ("ASE_OBJ", 2), + ("ASE_REQ", 2), + ("ASE_SPD", 1), + ("ASE_TSS", 1), + ("ATE_COV", 3), + ("ATE_DPT", 4), + ("ATE_FUN", 2), + ("ATE_IND", 3), + ("AVA_VAN", 5), + }, +} + +security_level_csv_scan = r"EAL[1-7]\+?" + + +REGEXEC_SEP = "[ ,;\\[\\]”\"')(.]" +MATCH_START = "(?P" +MATCH_END = ")" +REGEXEC_SEP_START = f"(?:^|{REGEXEC_SEP})" +REGEXEC_SEP_END = f"(?:$|{REGEXEC_SEP})" + + +SERVICE_PACK_RE = re.compile(r"(?:sp|service pack)\s{0,1}\d{1,2}", re.IGNORECASE) +RELEASE_RE = re.compile(r"(?:r|release)\s{0,1}\d{1,2}", re.IGNORECASE) +PLATFORM_REGEXES = { + "linux": re.compile(r"linux", re.IGNORECASE), + "mac_os": re.compile(r"mac\s?os\s?x?", re.IGNORECASE), + "windows": re.compile(r"windows", re.IGNORECASE), + "android": re.compile(r"android", re.IGNORECASE), + "ios": re.compile(r"(ios|iphone os)", re.IGNORECASE), +} + +FIPS_ALGS_IN_TABLE = r"(?:#[CcAa]?\s?|(?:Cert)\.?[^. ]*?\s?)(?:[CcAa]\s)?(?P\d+)" +FIPS_LIST_OF_TABLES = re.compile(r"^(?:(?:[Tt]able\s|[Ll]ist\s)(?:[Oo]f\s))[Tt]ables[\s\S]+?\f", re.MULTILINE) + + +def _load(): + script_dir = Path(__file__).parent + filepath = script_dir / "rules.yaml" + with Path(filepath).open("r") as file: + loaded = yaml.load(file, Loader=yaml.FullLoader) + return loaded + + +def _process(obj): + if isinstance(obj, dict): + return {k: _process(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [ + re.compile( + REGEXEC_SEP_START + MATCH_START + rule + MATCH_END + REGEXEC_SEP_END, + re.MULTILINE, + ) + for rule in obj + ] + + +rules = _load() + +cc_rules = {} +for rule_group in rules["cc_rules"]: + cc_rules[rule_group] = _process(rules[rule_group]) + +fips_rules = {} +for rule_group in rules["fips_rules"]: + fips_rules[rule_group] = _process(rules[rule_group]) + + +PANDAS_KEYWORDS_CATEGORIES: Final[list[str]] = [ + "symmetric_crypto", + "asymmetric_crypto", + "pq_crypto", + "hash_function", + "crypto_scheme", + "crypto_protocol", + "randomness", + "cipher_mode", + "ecc_curve", + "crypto_engine", + "crypto_library", +] diff --git a/src/sec_certs/cli.py b/src/sec_certs/cli.py new file mode 100644 index 00000000..a25b55ee --- /dev/null +++ b/src/sec_certs/cli.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import logging +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Callable + +import click + +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.dataset import CCDataset, FIPSDataset +from sec_certs.utils.helpers import warn_if_missing_poppler, warn_if_missing_tesseract + +logger = logging.getLogger(__name__) + +EXIT_CODE_NOK: int = 1 +EXIT_CODE_OK: int = 0 + + +@dataclass +class ProcessingStep: + name: str + processing_function_name: str + preconditions: list[str] = field(default_factory=list) + precondition_error_msg: str | None = field(default=None) + pre_callback_func: Callable | None = field(default=None) + + def run(self, dset: CCDataset | FIPSDataset) -> None: + for condition in self.preconditions: + if not getattr(dset.state, condition): + err_msg = ( + self.precondition_error_msg + if self.precondition_error_msg + else f"Error, precondition to run {self.name} not met, exiting." + ) + click.echo(err_msg, err=True) + sys.exit(EXIT_CODE_NOK) + if self.pre_callback_func: + self.pre_callback_func() + + getattr(dset, self.processing_function_name)() + + +def warn_missing_libs(): + warn_if_missing_poppler() + warn_if_missing_tesseract() + + +def build_or_load_dataset( + framework: str, + inputpath: Path | None, + to_build: bool, + outputpath: Path = constants.DUMMY_NONEXISTING_PATH, +) -> CCDataset | FIPSDataset: + constructor: type[CCDataset] | type[FIPSDataset] = CCDataset if framework == "cc" else FIPSDataset + dset: CCDataset | FIPSDataset + + if to_build: + if inputpath: + print( + f"Warning: you wanted to build a dataset but you provided one in JSON -- that will be ignored. New one will be constructed at: {outputpath}" + ) + dset = constructor( + certs={}, + root_dir=outputpath, + name=framework + "_dataset", + description=f"Full {framework} dataset snapshot {datetime.now().date()}", + ) + dset.get_certs_from_web() + else: + if inputpath: + dset = constructor.from_json(inputpath) + if outputpath and dset.root_dir != outputpath: + print( + "Warning: you provided both input and output paths. The dataset from input path will get copied to output path." + ) + dset.copy_dataset(outputpath) + else: + click.echo( + "Error: If you do not use 'build' action, you must provide --input parameter to point to an existing dataset.", + err=True, + ) + sys.exit(EXIT_CODE_NOK) + + return dset + + +@click.command() +@click.argument( + "framework", + required=True, + nargs=1, + type=click.Choice(["cc", "fips"], case_sensitive=False), +) +@click.argument( + "actions", + required=True, + nargs=-1, + type=click.Choice(["all", "build", "process-aux-dsets", "download", "convert", "analyze"], case_sensitive=False), +) +@click.option( + "-o", + "--output", + type=click.Path(file_okay=False, dir_okay=True, writable=True, readable=True, resolve_path=True), + help="Path where the output of the experiment will be stored. May overwrite existing content.", + default=Path("./dataset/"), + show_default=True, +) +@click.option( + "-c", + "--config", + "configpath", + default=None, + type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), + help="Path to your own config yaml file that will override the default one.", +) +@click.option( + "-i", + "--input", + "inputpath", + type=click.Path(file_okay=True, dir_okay=False, writable=True, readable=True), + help="If set, the actions will be performed on a CC dataset loaded from JSON from the input path.", +) +@click.option("-q", "--quiet", is_flag=True, help="If set, will not print to stdout") +def main( + framework: str, + actions: list[str], + outputpath: Path, + configpath: str | None, + inputpath: Path | None, + silent: bool, +): + try: + file_handler = logging.FileHandler(config.log_filepath) + stream_handler = logging.StreamHandler(sys.stderr) + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + file_handler.setFormatter(formatter) + stream_handler.setFormatter(formatter) + handlers: list[logging.StreamHandler] = [file_handler] if silent else [file_handler, stream_handler] + logging.basicConfig(level=logging.INFO, handlers=handlers) + start = datetime.now() + + if configpath: + try: + config.load(Path(configpath)) + except FileNotFoundError: + click.echo("Error: Bad path to configuration file", err=True) + sys.exit(EXIT_CODE_NOK) + except ValueError as e: + click.echo(f"Error: Bad format of configuration file: {e}", err=True) + sys.exit(EXIT_CODE_NOK) + + actions_set = ( + {"build", "process-aux-dsets", "download", "convert", "analyze", "maintenances"} + if "all" in actions + else set(actions) + ) + + dset = build_or_load_dataset(framework, inputpath, "build" in actions_set, outputpath) + aux_dsets_to_handle = "PP, Maintenance updates" if framework == "cc" else "Algorithms" + aux_dsets_to_handle += "CPE, CVE" + analysis_pre_callback = None + + steps = [ + ProcessingStep( + "process-aux-dsets", + "process_auxillary_datasets", + preconditions=["meta_sources_parsed"], + precondition_error_msg=f"Error: You want to process the auxillary datasets: {aux_dsets_to_handle} , but the data from cert. framework website was not parsed. You must use 'build' action first.", + pre_callback_func=None, + ), + ProcessingStep( + "download", + "download_all_artifacts", + preconditions=["meta_sources_parsed"], + precondition_error_msg="Error: You want to download all artifacts, but the data from the cert. framework website was not parsed. You must use 'build' action first.", + pre_callback_func=None, + ), + ProcessingStep( + "convert", + "convert_all_pdfs", + preconditions=["pdfs_downloaded"], + precondition_error_msg="Error: You want to convert pdfs -> txt, but the pdfs were not downloaded. You must use 'download' action first.", + pre_callback_func=warn_missing_libs, + ), + ProcessingStep( + "analyze", + "analyze_certificates", + preconditions=["pdfs_converted", "auxillary_datasets_processed"], + precondition_error_msg="Error: You want to process txt documents of certificates, but pdfs were not converted. You must use 'convert' action first.", + pre_callback_func=analysis_pre_callback, + ), + ] + + processing_step: ProcessingStep + for processing_step in [x for x in steps if x in actions_set]: + processing_step.run(dset) + + end = datetime.now() + logger.info(f"The computation took {(end-start)} seconds.") + except Exception: + return EXIT_CODE_NOK + return EXIT_CODE_OK + + +if __name__ == "__main__": + main() diff --git a/src/sec_certs/config/__init__.py b/src/sec_certs/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sec_certs/config/configuration.py b/src/sec_certs/config/configuration.py new file mode 100644 index 00000000..43cfe698 --- /dev/null +++ b/src/sec_certs/config/configuration.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import jsonschema +import yaml + + +class Configuration: + def load(self, filepath: str | Path) -> None: + with Path(filepath).open("r") as file: + state = yaml.load(file, Loader=yaml.FullLoader) + + script_dir = Path(__file__).parent + + with (Path(script_dir) / "settings-schema.json").open("r") as file: + schema = json.loads(file.read()) + + try: + jsonschema.validate(state, schema) + except jsonschema.exceptions.ValidationError as e: + print(f"{e}\n\nIn file {filepath}") + + for k, v in state.items(): + setattr(self, k, v) + + def __getattribute__(self, key: str) -> Any: + res = object.__getattribute__(self, key) + if isinstance(res, dict) and "value" in res: + return res["value"] + return object.__getattribute__(self, key) + + def get_desription(self, key: str) -> str | None: + res = object.__getattribute__(self, key) + if isinstance(res, dict) and "description" in res: + return res["description"] + return None + + +DEFAULT_CONFIG_PATH = Path(__file__).parent / "settings.yaml" +config = Configuration() +config.load(DEFAULT_CONFIG_PATH) diff --git a/src/sec_certs/config/settings-schema.json b/src/sec_certs/config/settings-schema.json new file mode 100644 index 00000000..a31f6b3f --- /dev/null +++ b/src/sec_certs/config/settings-schema.json @@ -0,0 +1,182 @@ +{ + "title": "settings for sec-certs", + "type": "object", + "definitions": { + "settings_string_entry": { + "required": [ + "description", + "value" + ], + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "settings_boolean_entry": { + "type": "object", + "required": [ + "description", + "value" + ], + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "boolean" + } + } + }, + "settings_number_entry": { + "type": "object", + "required": [ + "description", + "value" + ], + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "number" + } + } + }, + "settings_url_entry": { + "type": "object", + "required": [ + "description", + "value" + ], + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "string", + "format": "uri", + "pattern": "^(https?|http?)://", + "minLength": 1, + "maxLength": 255 + } + } + } + }, + "properties": { + "log_filepath": { + "$ref": "#/definitions/settings_string_entry" + }, + "always_false_positive_fips_cert_id_threshold": { + "$ref": "#/definitions/settings_number_entry" + }, + "year_difference_between_validations": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 0 + } + } + } + ] + }, + "n_threads": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 1 + } + } + } + ] + }, + "cpe_n_matching_threshold": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 0, + "maximum": 100 + } + } + } + ] + }, + "cpe_n_max_matches": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "exclusiveMinimum": 0 + } + } + } + ] + }, + "cc_latest_snapshot": { + "$ref": "#/definitions/settings_url_entry" + }, + "cc_maintenances_latest_snapshot": { + "$ref": "#/definitions/settings_url_entry" + }, + "pp_latest_snapshot": { + "$ref": "#/definitions/settings_url_entry" + }, + "ignore_first_page": { + "$ref": "#/definitions/settings_boolean_entry" + }, + "cert_threshold": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 0 + } + } + } + ] + }, + "fips_latest_snapshot": { + "$ref": "#/definitions/settings_url_entry" + }, + "enable_progress_bars": { + "$ref": "#/definitions/settings_boolean_entry" + } + }, + "required": [ + "log_filepath", + "always_false_positive_fips_cert_id_threshold", + "year_difference_between_validations", + "n_threads", + "cpe_matching_threshold", + "cpe_n_max_matches", + "cc_latest_snapshot", + "cc_maintenances_latest_snapshot", + "pp_latest_snapshot", + "ignore_first_page", + "cert_threshold", + "fips_latest_snapshot", + "enable_progress_bars" + ] +} \ No newline at end of file diff --git a/src/sec_certs/config/settings.yaml b/src/sec_certs/config/settings.yaml new file mode 100644 index 00000000..8d645758 --- /dev/null +++ b/src/sec_certs/config/settings.yaml @@ -0,0 +1,59 @@ +--- +log_filepath: + description: Path to the file, relative to working directory, where the log will be stored + value: ./cert_processing_log.txt +always_false_positive_fips_cert_id_threshold: + description: + During validation we don't connect certificates with number lower than + _this_ to connections due to these numbers being typically false positives + value: 40 +year_difference_between_validations: + description: + During validation we don't connect certificates with validation dates + difference higher than _this_ + value: 7 +n_threads: + description: How many threads to use for parallel computations + value: 8 +cpe_matching_threshold: + description: Level of required string similarity between CPE and certificate name on CC CPE matching, 0-100. Lower values yield more false negatives, higher values more false positives + value: 92 +cpe_n_max_matches: + description: Maximum number of candidate CPE items that may be related to given certificate, >0 + value: 99 +cc_latest_snapshot: + description: URL from where to fetch the latest snapshot of fully processed CC dataset + value: https://seccerts.org/cc/dataset.json +cc_maintenances_latest_snapshot: + description: URL from where to fetch the latest snapshot of CC maintenance updates + value: https://seccerts.org/cc/maintenance_updates.json +pp_latest_snapshot: + description: URL from where to fetch the latest snapshot of the PP dataset + value: https://seccerts.org/static/pp.json +ignore_first_page: + description: During keyword search, first page usually contains addresses - ignore it. + value: true +cert_threshold: + description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty. + value: 5 +fips_latest_snapshot: + description: URL for the latest snapshot of FIPS dataset + value: https://seccerts.org/fips/dataset.json +fips_iut_dataset: + description: URL for the dataset of FIPS IUT data + value: https://seccerts.org/fips/iut/dataset.json +fips_iut_latest_snapshot: + description: URL for the latest snapshot of FIPS IUT data + value: https://seccerts.org/fips/iut/latest.json +fips_mip_dataset: + description: URL for the dataset of FIPS MIP data + value: https://seccerts.org/fips/mip/dataset.json +fips_mip_latest_snapshot: + description: URL for the latest snapshot of FIPS MIP data + value: https://seccerts.org/fips/mip/latest.json +minimal_token_length: + description: Minimal length of a string that will be considered as a token during keyword extraction in CVE matching + value: 3 +enable_progress_bars: + description: Whether to enable pretty-printed progress bars while processing. + value: true diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py new file mode 100644 index 00000000..f71bb3c3 --- /dev/null +++ b/src/sec_certs/constants.py @@ -0,0 +1,121 @@ +import re +from pathlib import Path + +DUMMY_NONEXISTING_PATH = Path("/this/is/dummy/nonexisting/path") + +RESPONSE_OK = 200 +RETURNCODE_OK = "ok" +RETURNCODE_NOK = "nok" +REQUEST_TIMEOUT = 10 + +MIN_CORRECT_CERT_SIZE = 5000 + +MIN_FIPS_HTML_SIZE = 64000 + +MIN_CC_HTML_SIZE = 5000000 +MIN_CC_CSV_SIZE = 700000 +MIN_CC_PP_DATASET_SIZE = 2500000 + +CPE_VERSION_NA = "-" + +RELEASE_CANDIDATE_REGEX: re.Pattern = re.compile(r"rc\d{0,2}$", re.IGNORECASE) + +FIPS_BASE_URL = "https://csrc.nist.gov" +FIPS_CMVP_URL = FIPS_BASE_URL + "/projects/cryptographic-module-validation-program" +FIPS_CAVP_URL = FIPS_BASE_URL + "/projects/Cryptographic-Algorithm-Validation-Program" +FIPS_MODULE_URL = FIPS_CMVP_URL + "/certificate/{}" +FIPS_ALG_SEARCH_URL = FIPS_CAVP_URL + "/validation-search?searchMode=implementation&page=" +FIPS_SP_URL = "https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{}.pdf" +FIPS_ACTIVE_MODULES_URL = ( + FIPS_CMVP_URL + "/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0" +) +FIPS_HISTORICAL_MODULES_URL = ( + FIPS_CMVP_URL + "/validated-modules/search?SearchMode=Advanced&CertificateStatus=Historical&ValidationYear=0" +) +FIPS_REVOKED_MODULES_URL = ( + FIPS_CMVP_URL + "/validated-modules/search?SearchMode=Advanced&CertificateStatus=Revoked&ValidationYear=0" +) +FIPS_ALG_URL = FIPS_CAVP_URL + "/details?source={}&number={}" +FIPS_IUT_URL = "https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/IUT-List" +FIPS_MIP_URL = ( + "https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/Modules-In-Process-List" +) + +FIPS_DOWNLOAD_DELAY = 1 + +FIPS_MIP_STATUS_RE = re.compile(r"^(?P[a-zA-Z ]+?) +\((?P\d{1,2}/\d{1,2}/\d{4})\)$") + +TAG_CERT_ID = "cert_id" +TAG_CC_SECURITY_LEVEL = "cc_security_level" +TAG_CC_VERSION = "cc_version" +TAG_CERT_LAB = "cert_lab" +TAG_CERT_ITEM = "cert_item" +TAG_CERT_ITEM_VERSION = "cert_item_version" +TAG_DEVELOPER = "developer" +TAG_REFERENCED_PROTECTION_PROFILES = "ref_protection_profiles" +TAG_HEADER_MATCH_RULES = "match_rules" + +FILE_ERRORS_STRATEGY = "surrogateescape" +MAX_ALLOWED_MATCH_LENGTH = 300 +LINE_SEPARATOR = " " + +GARBAGE_LINES_THRESHOLD = 30 +GARBAGE_SIZE_THRESHOLD = 1000 +GARBAGE_AVG_LLEN_THRESHOLD = 10 +GARBAGE_EVERY_SECOND_CHAR_THRESHOLD = 15 +GARBAGE_ALPHA_CHARS_THRESHOLD = 0.5 + +CC_AUSTRALIA_BASE_URL = "https://www.cyber.gov.au" +CC_AUSTRALIA_CERTIFIED_URL = ( + CC_AUSTRALIA_BASE_URL + "/acsc/view-all-content/programs/australian-information-security-evaluation-program" +) +CC_CANADA_CERTIFIED_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/certified-products" +CC_CANADA_INEVAL_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/products-evaluation" +CC_ANSSI_BASE_URL = "https://www.ssi.gouv.fr" +CC_ANSSI_CERTIFIED_URL = CC_ANSSI_BASE_URL + "/en/products/certified-products/" +CC_BSI_BASE_URL = "https://www.bsi.bund.de/" +CC_BSI_CERTIFIED_URL = CC_BSI_BASE_URL + "EN2021/Topics/Certification/certified_products/certified_products_node.html" +CC_INDIA_CERTIFIED_URL = "https://www.commoncriteria-india.gov.in/product-certified" +CC_INDIA_ARCHIVED_URL = "https://www.commoncriteria-india.gov.in/archived-prod-cer" +CC_ITALY_BASE_URL = "https://www.ocsi.gov.it" +CC_ITALY_CERTIFIED_URL = CC_ITALY_BASE_URL + "/index.php/elenchi-certificazioni/prodotti-certificati.html" +CC_ITALY_INEVAL_URL = CC_ITALY_BASE_URL + "/index.php/elenchi-certificazioni/in-corso-di-valutazione.html" +CC_JAPAN_BASE_URL = "https://www.ipa.go.jp/security/jisec/jisec_e" +CC_JAPAN_CERT_BASE_URL = CC_JAPAN_BASE_URL + "/certified_products" +CC_JAPAN_CERTIFIED_URL = CC_JAPAN_BASE_URL + "/certified_products/certfy_list_e31.html" +CC_JAPAN_ARCHIVED_URL = CC_JAPAN_BASE_URL + "/certified_products/certfy_list_e_archive.html" +CC_JAPAN_INEVAL_URL = CC_JAPAN_BASE_URL + "/prdct_in_eval.html" +CC_MALAYSIA_BASE_URL = "https://iscb.cybersecurity.my" +CC_MALAYSIA_CERTIFIED_URL = ( + CC_MALAYSIA_BASE_URL + "/en/index.php/certification/product-certification/mycc/certified-products-and-systems" +) +CC_MALAYSIA_INEVAL_URL = ( + CC_MALAYSIA_BASE_URL + + "/en/index.php/certification/product-certification/mycc/list-of-products-and-systems-under-evaluation-or-maintenance" +) +CC_NETHERLANDS_BASE_URL = "https://www.tuv-nederland.nl/common-criteria" +CC_NETHERLANDS_CERTIFIED_URL = CC_NETHERLANDS_BASE_URL + "/certificates.html" +CC_NETHERLANDS_INEVAL_URL = CC_NETHERLANDS_BASE_URL + "/ongoing-certifications.html" +CC_NORWAY_CERTIFIED_URL = "https://sertit.no/certified-products/category1919.html" +CC_NORWAY_ARCHIVED_URL = "https://sertit.no/certified-products/product-archive/" +CC_KOREA_EN_URL = "https://itscc.kr/main/main.do?accessMode=home_en" +CC_KOREA_CERTIFIED_URL = "https://itscc.kr/certprod/list.do" +CC_KOREA_PRODUCT_URL = "https://itscc.kr/certprod/view.do?product_id={}&product_class=1" +CC_SINGAPORE_BASE_URL = "https://www.csa.gov.sg" +CC_SINGAPORE_CERTIFIED_URL = ( + CC_SINGAPORE_BASE_URL + "/Programmes/certification-and-labelling-schemes/csa-common-criteria/product-list" +) +CC_SINGAPORE_ARCHIVED_URL = ( + CC_SINGAPORE_BASE_URL + "/Programmes/certification-and-labelling-schemes/csa-common-criteria/product-archives" +) +CC_SPAIN_BASE_URL = "https://oc.ccn.cni.es" +CC_SPAIN_CERTIFIED_URL = CC_SPAIN_BASE_URL + "/en/certified-products/certified-products" +CC_SWEDEN_BASE_URL = "https://www.fmv.se" +CC_SWEDEN_CERTIFIED_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/certifikat-utgivna-av-csec/" +CC_SWEDEN_INEVAL_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/pagaende-certifieringar/" +CC_SWEDEN_ARCHIVED_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/arkiverade-certifikat-aldre-an-5-ar/" +CC_TURKEY_ARCHIVED_URL = "https://statik.tse.org.tr/upload/tr/dosya/icerikyonetimi/3300/03112021143434-2.pdf" +CC_USA_BASE_URL = "https://www.niap-ccevs.org" +CC_USA_CERTIFIED_URL = CC_USA_BASE_URL + "/Product/PCL.cfm" +CC_USA_INEVAL_URL = CC_USA_BASE_URL + "/Product/PINE.cfm" +CC_USA_ARCHIVED_URL = CC_USA_BASE_URL + "/Product/Archived.cfm" diff --git a/src/sec_certs/dataset/__init__.py b/src/sec_certs/dataset/__init__.py new file mode 100644 index 00000000..bace4ddd --- /dev/null +++ b/src/sec_certs/dataset/__init__.py @@ -0,0 +1,22 @@ +"""This package exposes Datasets of various Samples, both primary (Common Criteria, FIPS) and auxillary (CVEs, CPEs, ...)""" + +from sec_certs.dataset.common_criteria import CCDataset, CCDatasetMaintenanceUpdates +from sec_certs.dataset.cpe import CPEDataset +from sec_certs.dataset.cve import CVEDataset +from sec_certs.dataset.fips import FIPSDataset +from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset +from sec_certs.dataset.fips_iut import IUTDataset +from sec_certs.dataset.fips_mip import MIPDataset +from sec_certs.dataset.protection_profile import ProtectionProfileDataset + +__all__ = [ + "CCDataset", + "CCDatasetMaintenanceUpdates", + "CPEDataset", + "CVEDataset", + "FIPSDataset", + "FIPSAlgorithmDataset", + "IUTDataset", + "MIPDataset", + "ProtectionProfileDataset", +] diff --git a/src/sec_certs/dataset/common_criteria.py b/src/sec_certs/dataset/common_criteria.py new file mode 100644 index 00000000..6b7b4279 --- /dev/null +++ b/src/sec_certs/dataset/common_criteria.py @@ -0,0 +1,1674 @@ +from __future__ import annotations + +import itertools +import json +import locale +import shutil +import tempfile +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import ClassVar, Iterator + +import numpy as np +import pandas as pd +import requests +import tabula +from bs4 import BeautifulSoup, NavigableString, Tag + +import sec_certs.utils.sanitization +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.dataset.cpe import CPEDataset +from sec_certs.dataset.cve import CVEDataset +from sec_certs.dataset.dataset import AuxillaryDatasets, Dataset, logger +from sec_certs.dataset.protection_profile import ProtectionProfileDataset +from sec_certs.model.reference_finder import ReferenceFinder +from sec_certs.model.sar_transformer import SARTransformer +from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder +from sec_certs.sample.cc_certificate_id import CertificateId +from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.protection_profile import ProtectionProfile +from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, serialize +from sec_certs.utils import helpers as helpers +from sec_certs.utils import parallel_processing as cert_processing +from sec_certs.utils.sanitization import sanitize_navigable_string as sns + + +@dataclass +class CCAuxillaryDatasets(AuxillaryDatasets): + cpe_dset: CPEDataset | None = None + cve_dset: CVEDataset | None = None + pp_dset: ProtectionProfileDataset | None = None + mu_dset: CCDatasetMaintenanceUpdates | None = None + + +class CCDataset(Dataset[CommonCriteriaCert, CCAuxillaryDatasets], ComplexSerializableType): + """ + Class that holds CommonCriteriaCert. Serializable into json, pandas, dictionary. Conveys basic certificate manipulations + and dataset transformations. Many private methods that perform internal operations, feel free to exploit them. + """ + + def __init__( + self, + certs: dict[str, CommonCriteriaCert] = dict(), + root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH, + name: str | None = None, + description: str = "", + state: Dataset.DatasetInternalState | None = None, + auxillary_datasets: CCAuxillaryDatasets | None = None, + ): + self.certs = certs + self.timestamp = datetime.now() + self.sha256_digest = "not implemented" + self.name = name if name else type(self).__name__ + " dataset" + self.description = description if description else datetime.now().strftime("%d/%m/%Y %H:%M:%S") + self.state = state if state else self.DatasetInternalState() + + self.auxillary_datasets: CCAuxillaryDatasets = ( + auxillary_datasets if auxillary_datasets else CCAuxillaryDatasets() + ) + + self.root_dir = Path(root_dir) + + def to_pandas(self) -> pd.DataFrame: + """ + Return self serialized into pandas DataFrame + """ + df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=CommonCriteriaCert.pandas_columns) + df = df.set_index("dgst") + + df.not_valid_before = pd.to_datetime(df.not_valid_before, infer_datetime_format=True) + df.not_valid_after = pd.to_datetime(df.not_valid_after, infer_datetime_format=True) + df = df.astype( + {"category": "category", "status": "category", "scheme": "category", "cert_lab": "category"} + ).fillna(value=np.nan) + df = df.loc[ + ~df.manufacturer.isnull() + ] # Manually delete one certificate with None manufacturer (seems to have many blank fields) + + # Categorize EAL + df.eal = df.eal.fillna(value=np.nan) + df.eal = pd.Categorical(df.eal, categories=sorted(df.eal.dropna().unique().tolist()), ordered=True) + + # Introduce year when cert got valid + df["year_from"] = pd.DatetimeIndex(df.not_valid_before).year + + return df + + @property + def reports_dir(self) -> Path: + """ + Returns directory that holds files associated with certification reports + """ + return self.certs_dir / "reports" + + @property + def reports_pdf_dir(self) -> Path: + """ + Returns directory that holds PDFs associated with certification reports + """ + return self.reports_dir / "pdf" + + @property + def reports_txt_dir(self) -> Path: + """ + Returns directory that holds TXTs associated with certification reports + """ + return self.reports_dir / "txt" + + @property + def targets_dir(self) -> Path: + """ + Returns directory that holds files associated with security targets + """ + return self.certs_dir / "targets" + + @property + def targets_pdf_dir(self) -> Path: + """ + Returns directory that holds PDFs associated with security targets + """ + return self.targets_dir / "pdf" + + @property + def targets_txt_dir(self) -> Path: + """ + Returns directory that holds TXTs associated with security targets + """ + return self.targets_dir / "txt" + + @property + def pp_dataset_path(self) -> Path: + """ + Returns directory that holds files associated with Protection profiles + """ + return self.auxillary_datasets_dir / "pp_dataset.json" + + @property + def mu_dataset_dir(self) -> Path: + """ + Returns directory that holds dataset of maintenance updates + """ + return self.auxillary_datasets_dir / "maintenances" + + @property + def mu_dataset_path(self) -> Path: + """ + Returns json that holds the datase of maintenance updates + """ + return self.mu_dataset_dir / "maintenance_updates.json" + + BASE_URL: ClassVar[str] = "https://www.commoncriteriaportal.org" + + HTML_PRODUCTS_URL = { + "cc_products_active.html": BASE_URL + "/products/", + "cc_products_archived.html": BASE_URL + "/products/index.cfm?archived=1", + } + HTML_LABS_URL = {"cc_labs.html": BASE_URL + "/labs"} + CSV_PRODUCTS_URL = { + "cc_products_active.csv": BASE_URL + "/products/certified_products.csv", + "cc_products_archived.csv": BASE_URL + "/products/certified_products-archived.csv", + } + PP_URL = { + "cc_pp_active.html": BASE_URL + "/pps/", + "cc_pp_collaborative.html": BASE_URL + "/pps/collaborativePP.cfm?cpp=1", + "cc_pp_archived.html": BASE_URL + "/pps/index.cfm?archived=1", + } + PP_CSV = {"cc_pp_active.csv": BASE_URL + "/pps/pps.csv", "cc_pp_archived.csv": BASE_URL + "/pps/pps-archived.csv"} + + @property + def active_html_tuples(self) -> list[tuple[str, Path]]: + """ + Returns List Tuple[str, Path] where first element is name of html file and second element is its Path. + The files correspond to html files parsed from CC website that list all *active* certificates. + """ + return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if "active" in y] + + @property + def archived_html_tuples(self) -> list[tuple[str, Path]]: + """ + Returns List Tuple[str, Path] where first element is name of html file and second element is its Path. + The files correspond to html files parsed from CC website that list all *archived* certificates. + """ + return [(x, self.web_dir / y) for y, x in self.HTML_PRODUCTS_URL.items() if "archived" in y] + + @property + def active_csv_tuples(self) -> list[tuple[str, Path]]: + """ + Returns List Tuple[str, Path] where first element is name of csv file and second element is its Path. + The files correspond to csv files downloaded from CC website that list all *active* certificates. + """ + return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if "active" in y] + + @property + def archived_csv_tuples(self) -> list[tuple[str, Path]]: + """ + Returns List Tuple[str, Path] where first element is name of csv file and second element is its Path. + The files correspond to csv files downloaded from CC website that list all *archived* certificates. + """ + return [(x, self.web_dir / y) for y, x in self.CSV_PRODUCTS_URL.items() if "archived" in y] + + @classmethod + def from_web_latest(cls) -> CCDataset: + """ + Fetches the fresh snapshot of CCDataset from seccerts.org + """ + return cls.from_web(config.cc_latest_snapshot, "Downloading CC Dataset", "cc_latest_dataset.json") + + def _set_local_paths(self): + super()._set_local_paths() + + if self.auxillary_datasets.pp_dset: + self.auxillary_datasets.pp_dset.json_path = self.pp_dataset_path + + if self.auxillary_datasets.mu_dset: + self.auxillary_datasets.mu_dset.root_dir = self.mu_dataset_dir + + for cert in self: + cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir) + # TODO: This forgets to set local paths for other auxillary datasets + + def _merge_certs(self, certs: dict[str, CommonCriteriaCert], cert_source: str | None = None) -> None: + """ + Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates + """ + new_certs = {x.dgst: x for x in certs.values() if x not in self} + certs_to_merge = [x for x in certs.values() if x in self] + self.certs.update(new_certs) + + for crt in certs_to_merge: + self[crt.dgst].merge(crt, cert_source) + + logger.info(f"Added {len(new_certs)} new and merged further {len(certs_to_merge)} certificates to the dataset.") + + def _download_csv_html_resources(self, get_active: bool = True, get_archived: bool = True) -> None: + self.web_dir.mkdir(parents=True, exist_ok=True) + + html_items = [] + csv_items = [] + if get_active is True: + html_items.extend(self.active_html_tuples) + csv_items.extend(self.active_csv_tuples) + if get_archived is True: + html_items.extend(self.archived_html_tuples) + csv_items.extend(self.archived_csv_tuples) + + html_urls, html_paths = [x[0] for x in html_items], [x[1] for x in html_items] + csv_urls, csv_paths = [x[0] for x in csv_items], [x[1] for x in csv_items] + + logger.info("Downloading required csv and html files.") + helpers.download_parallel(html_urls, html_paths) + helpers.download_parallel(csv_urls, csv_paths) + + @serialize + def get_certs_from_web( + self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, get_archived: bool = True + ) -> None: + """ + Downloads CSV and HTML files that hold lists of certificates from common criteria website. Parses these files + and constructs CommonCriteriaCert objects, fills the dataset with those. + + :param bool to_download: If CSV and HTML files shall be downloaded (or existing files utilized), defaults to True + :param bool keep_metadata: If CSV and HTML files shall be kept on disk after download, defaults to True + :param bool get_active: If active certificates shall be parsed, defaults to True + :param bool get_archived: If archived certificates shall be parsed, defaults to True + """ + if to_download is True: + self._download_csv_html_resources(get_active, get_archived) + + logger.info("Adding CSV certificates to CommonCriteria dataset.") + csv_certs = self._get_all_certs_from_csv(get_active, get_archived) + self._merge_certs(csv_certs, cert_source="csv") + + # Someway along the way, 3 certificates get lost. + logger.info("Adding HTML certificates to CommonCriteria dataset.") + html_certs = self._get_all_certs_from_html(get_active, get_archived) + self._merge_certs(html_certs, cert_source="html") + + logger.info(f"The resulting dataset has {len(self)} certificates.") + + if not keep_metadata: + shutil.rmtree(self.web_dir) + + self._set_local_paths() + self.state.meta_sources_parsed = True + + def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> dict[str, CommonCriteriaCert]: + """ + Creates dictionary of new certificates from csv sources. + """ + csv_sources = list(self.CSV_PRODUCTS_URL.keys()) + csv_sources = [x for x in csv_sources if "active" not in x or get_active] + csv_sources = [x for x in csv_sources if "archived" not in x or get_archived] + + new_certs = {} + for file in csv_sources: + partial_certs = self._parse_single_csv(self.web_dir / file) + logger.info(f"Parsed {len(partial_certs)} certificates from: {file}") + new_certs.update(partial_certs) + return new_certs + + @staticmethod + def _parse_single_csv(file: Path) -> dict[str, CommonCriteriaCert]: + """ + Using pandas, this parses a single CSV file. + """ + + def map_ip_to_hostname(url: str) -> str: + if not url: + return url + tokens = url.split("/") + relative_path = "/" + "/".join(tokens[3:]) + return CCDataset.BASE_URL + relative_path + + def _get_primary_key_str(row: Tag): + prim_key = row["category"] + row["cert_name"] + row["report_link"] + return prim_key + + if "active" in str(file): + cert_status = "active" + else: + cert_status = "archived" + + csv_header = [ + "category", + "cert_name", + "manufacturer", + "scheme", + "security_level", + "protection_profiles", + "not_valid_before", + "not_valid_after", + "report_link", + "st_link", + "maintenance_date", + "maintenance_title", + "maintenance_report_link", + "maintenance_st_link", + ] + + # TODO: Now skipping bad lines, smarter heuristics to be built for dumb files + df = pd.read_csv(file, engine="python", encoding="windows-1252", on_bad_lines="skip") + df = df.rename(columns={x: y for (x, y) in zip(list(df.columns), csv_header)}) + + df["is_maintenance"] = ~df.maintenance_title.isnull() + df = df.fillna(value="") + + df[["not_valid_before", "not_valid_after", "maintenance_date"]] = df[ + ["not_valid_before", "not_valid_after", "maintenance_date"] + ].apply(pd.to_datetime) + + df["dgst"] = df.apply(lambda row: helpers.get_first_16_bytes_sha256(_get_primary_key_str(row)), axis=1) + + df_base = df.loc[~df.is_maintenance].copy() + df_main = df.loc[df.is_maintenance].copy() + + df_base.report_link = df_base.report_link.map(map_ip_to_hostname) + df_base.st_link = df_base.st_link.map(map_ip_to_hostname) + + df_main.maintenance_report_link = df_main.maintenance_report_link.map(map_ip_to_hostname) + df_main.maintenance_st_link = df_main.maintenance_st_link.map(map_ip_to_hostname) + + n_all = len(df_base) + n_deduplicated = len(df_base.drop_duplicates(subset=["dgst"])) + if (n_dup := n_all - n_deduplicated) > 0: + logger.warning(f"The CSV {file} contains {n_dup} duplicates by the primary key.") + + df_base = df_base.drop_duplicates(subset=["dgst"]) + df_main = df_main.drop_duplicates() + + profiles = { + x.dgst: { + ProtectionProfile(pp_name=y, pp_eal=None) + for y in sec_certs.utils.sanitization.sanitize_protection_profiles(x.protection_profiles) + } + for x in df_base.itertuples() + } + updates: dict[str, set] = {x.dgst: set() for x in df_base.itertuples()} + for x in df_main.itertuples(): + updates[x.dgst].add( + CommonCriteriaCert.MaintenanceReport( + x.maintenance_date.date(), x.maintenance_title, x.maintenance_report_link, x.maintenance_st_link + ) + ) + + certs = { + x.dgst: CommonCriteriaCert( + cert_status, + x.category, + x.cert_name, + x.manufacturer, + x.scheme, + x.security_level, + x.not_valid_before, + x.not_valid_after, + x.report_link, + x.st_link, + None, + None, + profiles.get(x.dgst, None), + updates.get(x.dgst, None), + None, + None, + None, + ) + for x in df_base.itertuples() + } + return certs + + def _get_all_certs_from_html(self, get_active: bool, get_archived: bool) -> dict[str, CommonCriteriaCert]: + """ + Prepares dictionary of certificates from all html files. + """ + html_sources = list(self.HTML_PRODUCTS_URL.keys()) + if get_active is False: + html_sources = [x for x in html_sources if "active" not in x] + if get_archived is False: + html_sources = [x for x in html_sources if "archived" not in x] + + new_certs = {} + for file in html_sources: + partial_certs = self._parse_single_html(self.web_dir / file) + logger.info(f"Parsed {len(partial_certs)} certificates from: {file}") + new_certs.update(partial_certs) + return new_certs + + @staticmethod + def _parse_single_html(file: Path) -> dict[str, CommonCriteriaCert]: + """ + Prepares a dictionary of certificates from a single html file. + """ + + def _get_timestamp_from_footer(footer): + locale.setlocale(locale.LC_ALL, "en_US") + footer_text = list(footer.stripped_strings)[0] + date_string = footer_text.split(",")[1:3] + time_string = footer_text.split(",")[3].split(" at ")[1] + formatted_datetime = date_string[0] + date_string[1] + " " + time_string + return datetime.strptime(formatted_datetime, " %B %d %Y %I:%M %p") + + def _parse_table( + soup: BeautifulSoup, cert_status: str, table_id: str, category_string: str + ) -> dict[str, CommonCriteriaCert]: + tables = soup.find_all("table", id=table_id) + + if not len(tables) <= 1: + raise ValueError( + f'The "{file.name}" was expected to contain <1 element. Instead, it contains: {len(tables)}
elements.' + ) + + if not tables: + return {} + + table = tables[0] + rows = list(table.find_all("tr")) + # header, footer = rows[0], rows[1] + body = rows[2:] + + # It's possible to obtain timestamp of the moment when the list was generated. It's identical for each table and should thus only be obtained once. Not necessarily in each table + # timestamp = _get_timestamp_from_footer(footer) + + # The following unused snippet extracts expected number of certs from the table + # caption_str = str(table.findAll('caption')) + # n_expected_certs = int(caption_str.split(category_string + ' – ')[1].split(' Certified Products')[0]) + + try: + table_certs = { + x.dgst: x + for x in [CommonCriteriaCert.from_html_row(row, cert_status, category_string) for row in body] + } + except ValueError as e: + raise ValueError(f"Bad html file: {file.name} ({str(e)})") from e + + return table_certs + + if "active" in str(file): + cert_status = "active" + else: + cert_status = "archived" + + cc_cat_abbreviations = ["AC", "BP", "DP", "DB", "DD", "IC", "KM", "MD", "MF", "NS", "OS", "OD", "DG", "TC"] + cc_table_ids = ["tbl" + x for x in cc_cat_abbreviations] + cc_categories = [ + "Access Control Devices and Systems", + "Boundary Protection Devices and Systems", + "Data Protection", + "Databases", + "Detection Devices and Systems", + "ICs, Smart Cards and Smart Card-Related Devices and Systems", + "Key Management Systems", + "Mobility", + "Multi-Function Devices", + "Network and Network-Related Devices and Systems", + "Operating Systems", + "Other Devices and Systems", + "Products for Digital Signatures", + "Trusted Computing", + ] + cat_dict = {x: y for (x, y) in zip(cc_table_ids, cc_categories)} + + with file.open("r") as handle: + soup = BeautifulSoup(handle, "html5lib") + + certs = {} + for key, val in cat_dict.items(): + certs.update(_parse_table(soup, cert_status, key, val)) + + return certs + + def _download_all_artifacts_body(self, fresh: bool = True) -> None: + self._download_reports(fresh) + self._download_targets(fresh) + + def _download_reports(self, fresh: bool = True) -> None: + self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) + certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh) and x.report_link] + + if fresh: + logger.info("Downloading PDFs of CC certification reports.") + if not fresh and certs_to_process: + logger.info( + f"Downloading {len(certs_to_process)} PDFs of CC certification reports for which previous download failed." + ) + + cert_processing.process_parallel( + CommonCriteriaCert.download_pdf_report, + certs_to_process, + config.n_threads, + progress_bar_desc="Downloading PDFs of CC certification reports", + ) + + def _download_targets(self, fresh: bool = True) -> None: + self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) + certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh)] + + if fresh: + logger.info("Downloading PDFs of CC security targets.") + if not fresh and certs_to_process: + logger.info( + f"Downloading {len(certs_to_process)} PDFs of CC security targets for which previous download failed.." + ) + + cert_processing.process_parallel( + CommonCriteriaCert.download_pdf_st, + certs_to_process, + config.n_threads, + progress_bar_desc="Downloading PDFs of CC security targets", + ) + + def _convert_reports_to_txt(self, fresh: bool = True) -> None: + self.reports_txt_dir.mkdir(parents=True, exist_ok=True) + certs_to_process = [x for x in self if x.state.report_is_ok_to_convert(fresh)] + + if fresh: + logger.info("Converting PDFs of certification reports to txt.") + if not fresh and certs_to_process: + logger.info( + f"Converting {len(certs_to_process)} PDFs of certification reports to txt for which previous conversion failed." + ) + + cert_processing.process_parallel( + CommonCriteriaCert.convert_report_pdf, + certs_to_process, + config.n_threads, + progress_bar_desc="Converting PDFs of certification reports to txt", + ) + + def _convert_targets_to_txt(self, fresh: bool = True) -> None: + self.targets_txt_dir.mkdir(parents=True, exist_ok=True) + certs_to_process = [x for x in self if x.state.st_is_ok_to_convert(fresh)] + + if fresh: + logger.info("Converting PDFs of security targets to txt.") + if not fresh and certs_to_process: + logger.info( + f"Converting {len(certs_to_process)} PDFs of security targets to txt for which previous conversion failed." + ) + + cert_processing.process_parallel( + CommonCriteriaCert.convert_st_pdf, + certs_to_process, + config.n_threads, + progress_bar_desc="Converting PDFs of security targets to txt", + ) + + def _convert_all_pdfs_body(self, fresh: bool = True) -> None: + self._convert_reports_to_txt(fresh) + self._convert_targets_to_txt(fresh) + + def _extract_report_metadata(self) -> None: + logger.info("Extracting report metadata") + certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_report_pdf_metadata, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting report metadata", + ) + self.update_with_certs(processed_certs) + + def _extract_targets_metadata(self) -> None: + logger.info("Extracting target metadata") + certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_st_pdf_metadata, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting target metadata", + ) + self.update_with_certs(processed_certs) + + def _extract_pdf_metadata(self) -> None: + self._extract_report_metadata() + self._extract_targets_metadata() + + def _extract_report_frontpage(self) -> None: + logger.info("Extracting report frontpages") + certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_report_pdf_frontpage, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting report frontpages", + ) + self.update_with_certs(processed_certs) + + def _extract_targets_frontpage(self) -> None: + logger.info("Extracting target frontpages") + certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_st_pdf_frontpage, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting target frontpages", + ) + self.update_with_certs(processed_certs) + + def _extract_pdf_frontpage(self) -> None: + self._extract_report_frontpage() + self._extract_targets_frontpage() + + def _extract_report_keywords(self) -> None: + logger.info("Extracting report keywords") + certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_report_pdf_keywords, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting report keywords", + ) + self.update_with_certs(processed_certs) + + def _extract_targets_keywords(self) -> None: + logger.info("Extracting target keywords") + certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + CommonCriteriaCert.extract_st_pdf_keywords, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting target keywords", + ) + self.update_with_certs(processed_certs) + + def _extract_pdf_keywords(self) -> None: + self._extract_report_keywords() + self._extract_targets_keywords() + + def extract_data(self) -> None: + logger.info("Extracting various data from certification artifacts") + self._extract_pdf_metadata() + self._extract_pdf_frontpage() + self._extract_pdf_keywords() + + def _compute_cert_labs(self) -> None: + logger.info("Computing heuristics: Deriving information about laboratories involved in certification.") + certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()] + for cert in certs_to_process: + cert.compute_heuristics_cert_lab() + + def _compute_normalized_cert_ids(self) -> None: + logger.info("Computing heuristics: Deriving information about certificate ids from artifacts.") + for cert in self: + cert.compute_heuristics_cert_id() + + def _compute_transitive_vulnerabilities(self): + logger.info("omputing heuristics: computing transitive vulnerabilities in referenc(ed/ing) certificates.") + transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: cert.heuristics.cert_id) + transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.report_references) + + for dgst in self.certs: + transitive_cve = transitive_cve_finder.predict_single_cert(dgst) + + self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves + self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves + + def _compute_heuristics(self) -> None: + self._compute_normalized_cert_ids() + super()._compute_heuristics() + self._compute_cert_labs() + self._compute_sars() + + def _compute_sars(self) -> None: + logger.info("Computing heuristics: Computing SARs") + transformer = SARTransformer().fit(self.certs.values()) + for cert in self: + cert.heuristics.extracted_sars = transformer.transform_single_cert(cert) + + def _compute_references(self) -> None: + def ref_lookup(kw_attr): + def func(cert): + kws = getattr(cert.pdf_data, kw_attr) + if not kws: + return set() + res = set() + for scheme, matches in kws["cc_cert_id"].items(): + for match in matches.keys(): + try: + canonical = CertificateId(scheme, match).canonical + res.add(canonical) + except Exception: + res.add(match) + return res + + return func + + logger.info("omputing heuristics: references between certificates.") + for ref_source in ("report", "st"): + kw_source = f"{ref_source}_keywords" + dep_attr = f"{ref_source}_references" + + finder = ReferenceFinder() + finder.fit(self.certs, lambda cert: cert.heuristics.cert_id, ref_lookup(kw_source)) # type: ignore + + for dgst in self.certs: + setattr(self.certs[dgst].heuristics, dep_attr, finder.predict_single_cert(dgst, keep_unknowns=False)) + + def process_auxillary_datasets(self, download_fresh: bool = False) -> None: + """ + Processes all auxillary datasets needed during computation. On top of base-class processing, + CC handles protection profiles and maintenance updates. + """ + super().process_auxillary_datasets(download_fresh) + self.auxillary_datasets.pp_dset = self.process_protection_profiles(to_download=download_fresh) + self.auxillary_datasets.mu_dset = self.process_maintenance_updates(to_download=download_fresh) + + @serialize + def process_protection_profiles( + self, to_download: bool = True, keep_metadata: bool = True + ) -> ProtectionProfileDataset: + """ + Downloads new snapshot of dataset with processed protection profiles (if it doesn't exist) and links PPs + with certificates within self. Assigns PPs to all certificates + + :param bool to_download: If dataset should be downloaded or fetched from json, defaults to True + :param bool keep_metadata: If json related to the PP dataset should be kept on drive, defaults to True + :raises RuntimeError: When building of PPDataset fails + """ + logger.info("Processing protection profiles.") + + self.auxillary_datasets_dir.mkdir(parents=True, exist_ok=True) + + if to_download or not self.pp_dataset_path.exists(): + pp_dataset = ProtectionProfileDataset.from_web(self.pp_dataset_path) + else: + pp_dataset = ProtectionProfileDataset.from_json(self.pp_dataset_path) + + for cert in self: + if cert.protection_profiles is None: + raise RuntimeError("Building of the dataset probably failed - this should not be happening.") + cert.protection_profiles = {pp_dataset.pps.get((x.pp_name, x.pp_link), x) for x in cert.protection_profiles} + + if not keep_metadata: + self.pp_dataset_path.unlink() + + return pp_dataset + + def process_maintenance_updates(self, to_download: bool = True) -> CCDatasetMaintenanceUpdates: + """ + Downloads or loads from json a dataset of maintenance updates. Runs analysis on that dataset if it's not completed. + :return CCDatasetMaintenanceUpdates: the resulting dataset of maintenance updates + """ + + logger.info("Processing maintenace updates") + self.mu_dataset_dir.mkdir(parents=True, exist_ok=True) + + if to_download or not self.mu_dataset_path.exists(): + maintained_certs: list[CommonCriteriaCert] = [x for x in self if x.maintenance_updates] + updates = list( + itertools.chain.from_iterable( + CommonCriteriaMaintenanceUpdate.get_updates_from_cc_cert(x) for x in maintained_certs + ) + ) + update_dset = CCDatasetMaintenanceUpdates( + {x.dgst: x for x in updates}, root_dir=self.mu_dataset_dir, name="maintenance_updates" + ) + else: + update_dset = CCDatasetMaintenanceUpdates.from_json(self.mu_dataset_path) + + if not update_dset.state.artifacts_downloaded: + update_dset.download_all_artifacts() + if not update_dset.state.pdfs_converted: + update_dset.convert_all_pdfs() + if not update_dset.state.certs_analyzed: + update_dset.extract_data() + + return update_dset + + +class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType): + """ + Dataset of maintenance updates related to certificates of CCDataset dataset. + Should be used merely for actions related to Maintenance updates: download pdfs, convert pdfs, extract data from pdfs + """ + + # Quite difficult to achieve correct behaviour with MyPy here, opting for ignore + def __init__( + self, + certs: dict[str, CommonCriteriaMaintenanceUpdate] = dict(), # type: ignore + root_dir: Path = constants.DUMMY_NONEXISTING_PATH, + name: str = "dataset name", + description: str = "dataset_description", + state: CCDataset.DatasetInternalState | None = None, + ): + super().__init__(certs, root_dir, name, description, state) # type: ignore + self.state.meta_sources_parsed = True + + @property + def certs_dir(self) -> Path: + return self.root_dir + + def __iter__(self) -> Iterator[CommonCriteriaMaintenanceUpdate]: + yield from self.certs.values() # type: ignore + + def _compute_heuristics(self) -> None: + raise NotImplementedError + + def compute_related_cves(self) -> None: + raise NotImplementedError + + def process_auxillary_datasets(self, download_fresh: bool = False) -> None: + raise NotImplementedError + + def analyze_certificates(self) -> None: + raise NotImplementedError + + def get_certs_from_web( + self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, get_archived: bool = True + ) -> None: + raise NotImplementedError + + @classmethod + def from_json(cls, input_path: str | Path) -> CCDatasetMaintenanceUpdates: + input_path = Path(input_path) + with input_path.open("r") as handle: + dset = json.load(handle, cls=CustomJSONDecoder) + dset._root_dir = Path(input_path).parent + return dset + + def to_pandas(self) -> pd.DataFrame: + df = pd.DataFrame( + [x.pandas_tuple for x in self.certs.values()], columns=CommonCriteriaMaintenanceUpdate.pandas_columns + ) + df = df.set_index("dgst") + df.index.name = "dgst" + + df.maintenance_date = pd.to_datetime(df.maintenance_date, infer_datetime_format=True) + df = df.fillna(value=np.nan) + + return df + + @classmethod + def from_web_latest(cls) -> CCDatasetMaintenanceUpdates: + with tempfile.TemporaryDirectory() as tmp_dir: + dset_path = Path(tmp_dir) / "cc_maintenances_latest_dataset.json" + helpers.download_file(config.cc_maintenances_latest_snapshot, dset_path) + return cls.from_json(dset_path) + + def get_n_maintenances_df(self) -> pd.DataFrame: + """ + Returns a DataFrame with CommonCriteriaCert digest as an index, and number of registered maintenances as a value + """ + main_df = self.to_pandas() + main_df.maintenance_date = main_df.maintenance_date.dt.date + n_maintenances = ( + main_df.groupby("related_cert_digest").name.count().rename("n_maintenances").fillna(0).astype("int32") + ) + + n_maintenances.index.name = "dgst" + return n_maintenances + + def get_maintenance_dates_df(self) -> pd.DataFrame: + """ + Returns a DataFrame with CommonCriteriaCert digest as an index, and all the maintenance dates as a value. + """ + main_dates = self.to_pandas() + main_dates.maintenance_date = main_dates.maintenance_date.map(lambda x: [x]) + main_dates.index.name = "dgst" + return main_dates.groupby("related_cert_digest").maintenance_date.agg("sum").rename("maintenance_dates") + + +class CCSchemeDataset: + @staticmethod + def _download_page(url, session=None): + if session: + conn = session + else: + conn = requests + resp = conn.get(url, headers={"User-Agent": "seccerts.org"}) + if resp.status_code != requests.codes.ok: + raise ValueError(f"Unable to download: status={resp.status_code}") + return BeautifulSoup(resp.content, "html5lib") + + @staticmethod + def get_australia_in_evaluation(): + # TODO: Information could be expanded by following url. + soup = CCSchemeDataset._download_page(constants.CC_AUSTRALIA_CERTIFIED_URL) + header = soup.find("h2", text="Products in evaluation") + table = header.find_next_sibling("table") + results = [] + for tr in table.find_all("tr"): + tds = tr.find_all("td") + if not tds: + continue + cert = { + "vendor": sns(tds[0].text), + "product": sns(tds[1].text), + "url": constants.CC_AUSTRALIA_BASE_URL + tds[1].find("a")["href"], + "level": sns(tds[2].text), + } + results.append(cert) + return results + + @staticmethod + def get_canada_certified(): + soup = CCSchemeDataset._download_page(constants.CC_CANADA_CERTIFIED_URL) + tbody = soup.find("table").find("tbody") + results = [] + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + if not tds: + continue + cert = { + "product": sns(tds[0].text), + "vendor": sns(tds[1].text), + "level": sns(tds[2].text), + "certification_date": sns(tds[3].text), + } + results.append(cert) + return results + + @staticmethod + def get_canada_in_evaluation(): + soup = CCSchemeDataset._download_page(constants.CC_CANADA_INEVAL_URL) + tbody = soup.find("table").find("tbody") + results = [] + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + if not tds: + continue + cert = { + "product": sns(tds[0].text), + "vendor": sns(tds[1].text), + "level": sns(tds[2].text), + "cert_lab": sns(tds[3].text), + } + results.append(cert) + return results + + @staticmethod + def get_france_certified(): + # TODO: Information could be expanded by following product link. + base_soup = CCSchemeDataset._download_page(constants.CC_ANSSI_CERTIFIED_URL) + category_nav = base_soup.find("ul", class_="nav-categories") + results = [] + for li in category_nav.find_all("li"): + a = li.find("a") + url = a["href"] + category_name = sns(a.text) + soup = CCSchemeDataset._download_page(constants.CC_ANSSI_BASE_URL + url) + table = soup.find("table", class_="produits-liste cc") + if not table: + continue + tbody = table.find("tbody") + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + if not tds: + continue + cert = { + "product": sns(tds[0].text), + "vendor": sns(tds[1].text), + "level": sns(tds[2].text), + "id": sns(tds[3].text), + "certification_date": sns(tds[4].text), + "category": category_name, + "url": constants.CC_ANSSI_BASE_URL + tds[0].find("a")["href"], + } + results.append(cert) + return results + + @staticmethod + def get_germany_certified(): + # TODO: Information could be expanded by following url. + base_soup = CCSchemeDataset._download_page(constants.CC_BSI_CERTIFIED_URL) + category_nav = base_soup.find("ul", class_="no-bullet row") + results = [] + for li in category_nav.find_all("li"): + a = li.find("a") + url = a["href"] + category_name = sns(a.text) + soup = CCSchemeDataset._download_page(constants.CC_BSI_BASE_URL + url) + content = soup.find("div", class_="content").find("div", class_="column") + for table in content.find_all("table"): + tbody = table.find("tbody") + header = table.find_parent("div", class_="wrapperTable").find_previous_sibling("h2") + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + if len(tds) != 4: + continue + cert = { + "cert_id": sns(tds[0].text), + "product": sns(tds[1].text), + "vendor": sns(tds[2].text), + "certification_date": sns(tds[3].text), + "category": category_name, + "url": constants.CC_BSI_BASE_URL + tds[0].find("a")["href"], + } + if header is not None: + cert["subcategory"] = sns(header.text) + results.append(cert) + return results + + @staticmethod + def get_india_certified(): + pages = {0} + seen_pages = set() + results = [] + while pages: + page = pages.pop() + seen_pages.add(page) + url = constants.CC_INDIA_CERTIFIED_URL + f"?page={page}" + soup = CCSchemeDataset._download_page(url) + + # Update pages + pager = soup.find("ul", class_="pager") + for li in pager.find_all("li"): + try: + new_page = int(li.text) + except Exception: + continue + if new_page not in seen_pages: + pages.add(new_page) + + # Parse table + tbody = soup.find("div", class_="content").find("table").find("tbody") + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + if not tds: + continue + report_a = tds[5].find("a") + target_a = tds[6].find("a") + cert_a = tds[7].find("a") + cert = { + "serial_number": sns(tds[0].text), + "product": sns(tds[1].text), + "sponsor": sns(tds[2].text), + "developer": sns(tds[3].text), + "level": sns(tds[4].text), + "report_link": report_a["href"], + "report_name": sns(report_a.text), + "target_link": target_a["href"], + "target_name": sns(target_a.text), + "cert_link": cert_a["href"], + "cert_name": sns(cert_a.text), + } + results.append(cert) + return results + + @staticmethod + def get_india_archived(): + pages = {0} + seen_pages = set() + results = [] + while pages: + page = pages.pop() + seen_pages.add(page) + url = constants.CC_INDIA_ARCHIVED_URL + f"?page={page}" + soup = CCSchemeDataset._download_page(url) + + # Update pages + pager = soup.find("ul", class_="pager") + for li in pager.find_all("li"): + try: + new_page = int(li.text) + except Exception: + continue + if new_page not in seen_pages: + pages.add(new_page) + + # Parse table + tbody = soup.find("div", class_="content").find("table").find("tbody") + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + if not tds: + continue + report_a = tds[5].find("a") + target_a = tds[6].find("a") + cert_a = tds[7].find("a") + cert = { + "serial_number": sns(tds[0].text), + "product": sns(tds[1].text), + "sponsor": sns(tds[2].text), + "developer": sns(tds[3].text), + "level": sns(tds[4].text), + "report_link": report_a["href"], + "report_name": sns(report_a.text), + "target_link": target_a["href"], + "target_name": sns(target_a.text), + "cert_link": cert_a["href"], + "cert_name": sns(cert_a.text), + "certification_date": sns(tds[8].text), + } + results.append(cert) + return results + + @staticmethod + def get_italy_certified(): # noqa: C901 + soup = CCSchemeDataset._download_page(constants.CC_ITALY_CERTIFIED_URL) + div = soup.find("div", class_="certificati") + results = [] + for cert_div in div.find_all("div", recursive=False): + title = cert_div.find("h3").text + data_div = cert_div.find("div", class_="collapse") + cert = {"title": title} + for data_p in data_div.find_all("p"): + p_text = sns(data_p.text) + if ":" not in p_text: + continue + p_name, p_data = p_text.split(":") + p_data = p_data + p_link = data_p.find("a") + if "Fornitore" in p_name: + cert["supplier"] = p_data + elif "Livello di garanzia" in p_name: + cert["level"] = p_data + elif "Data emissione certificato" in p_name: + cert["certification_date"] = p_data + elif "Data revisione" in p_name: + cert["revision_date"] = p_data + elif "Rapporto di Certificazione" in p_name and p_link: + cert["report_link_it"] = constants.CC_ITALY_BASE_URL + p_link["href"] + elif "Certification Report" in p_name and p_link: + cert["report_link_en"] = constants.CC_ITALY_BASE_URL + p_link["href"] + elif "Traguardo di Sicurezza" in p_name and p_link: + cert["target_link"] = constants.CC_ITALY_BASE_URL + p_link["href"] + elif "Nota su" in p_name and p_link: + cert["vulnerability_note_link"] = constants.CC_ITALY_BASE_URL + p_link["href"] + elif "Nota di chiarimento" in p_name and p_link: + cert["clarification_note_link"] = constants.CC_ITALY_BASE_URL + p_link["href"] + results.append(cert) + return results + + @staticmethod + def get_italy_in_evaluation(): + soup = CCSchemeDataset._download_page(constants.CC_ITALY_INEVAL_URL) + div = soup.find("div", class_="valutazioni") + results = [] + for cert_div in div.find_all("div", recursive=False): + title = cert_div.find("h3").text + data_div = cert_div.find("div", class_="collapse") + cert = {"title": title} + for data_p in data_div.find_all("p"): + p_text = sns(data_p.text) + if ":" not in p_text: + continue + p_name, p_data = p_text.split(":") + p_data = p_data + if "Committente" in p_name: + cert["client"] = p_data + elif "Livello di garanzia" in p_name: + cert["level"] = p_data + elif "Tipologia prodotto" in p_name: + cert["product_type"] = p_data + results.append(cert) + return results + + @staticmethod + def get_japan_certified(): + # TODO: Information could be expanded by following toe link. + soup = CCSchemeDataset._download_page(constants.CC_JAPAN_CERTIFIED_URL) + table = soup.find("div", id="cert_list").find("table") + results = [] + trs = list(table.find_all("tr")) + for tr in trs: + tds = tr.find_all("td") + if not tds: + continue + if len(tds) == 6: + cert = { + "cert_id": sns(tds[0].text), + "supplier": sns(tds[1].text), + "toe_overseas_name": sns(tds[2].text), + "certification_date": sns(tds[3].text), + "claim": sns(tds[4].text), + } + toe_a = tds[2].find("a") + if toe_a and "href" in toe_a.attrs: + cert["toe_overseas_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] + results.append(cert) + if len(tds) == 1: + cert = results[-1] + cert["toe_japan_name"] = sns(tds[0].text) + toe_a = tds[0].find("a") + if toe_a and "href" in toe_a.attrs: + cert["toe_japan_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] + return results + + @staticmethod + def get_japan_archived(): + # TODO: Information could be expanded by following toe link. + soup = CCSchemeDataset._download_page(constants.CC_JAPAN_ARCHIVED_URL) + table = soup.find("table") + results = [] + trs = list(table.find_all("tr")) + for tr in trs: + tds = tr.find_all("td") + if not tds: + continue + if len(tds) == 6: + cert = { + "cert_id": sns(tds[0].text), + "supplier": sns(tds[1].text), + "toe_overseas_name": sns(tds[2].text), + "certification_date": sns(tds[3].text), + "claim": sns(tds[4].text), + } + toe_a = tds[2].find("a") + if toe_a and "href" in toe_a.attrs: + cert["toe_overseas_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] + results.append(cert) + if len(tds) == 1: + cert = results[-1] + cert["toe_japan_name"] = sns(tds[0].text) + toe_a = tds[0].find("a") + if toe_a and "href" in toe_a.attrs: + cert["toe_japan_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"] + return results + + @staticmethod + def get_japan_in_evaluation(): + # TODO: Information could be expanded by following toe link. + soup = CCSchemeDataset._download_page(constants.CC_JAPAN_INEVAL_URL) + table = soup.find("table") + results = [] + for tr in table.find_all("tr"): + tds = tr.find_all("td") + if not tds: + continue + toe_a = tds[1].find("a") + cert = { + "supplier": sns(tds[0].text), + "toe_name": sns(toe_a.text), + "toe_link": constants.CC_JAPAN_BASE_URL + "/" + toe_a["href"], + "claim": sns(tds[2].text), + } + results.append(cert) + return results + + @staticmethod + def get_malaysia_certified(): + soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_CERTIFIED_URL) + main_div = soup.find("div", attrs={"itemprop": "articleBody"}) + tables = main_div.find_all("table", recursive=False) + results = [] + for table in tables: + category_name = sns(table.find_previous_sibling("h3").text) + for tr in table.find_all("tr")[1:]: + tds = tr.find_all("td") + if len(tds) != 6: + continue + cert = { + "category": category_name, + "level": sns(tds[0].text), + "cert_id": sns(tds[1].text), + "certification_date": sns(tds[2].text), + "product": sns(tds[3].text), + "developer": sns(tds[4].text), + } + results.append(cert) + return results + + @staticmethod + def get_malaysia_in_evaluation(): + soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_INEVAL_URL) + main_div = soup.find("div", attrs={"itemprop": "articleBody"}) + tables = main_div.find_all("table", recursive=False) + results = [] + for table in tables: + category_name = sns(table.find_previous_sibling("h3").text) + for tr in table.find_all("tr")[1:]: + tds = tr.find_all("td") + if len(tds) != 5: + continue + cert = { + "category": category_name, + "level": sns(tds[0].text), + "project_id": sns(tds[1].text), + "toe_name": sns(tds[2].text), + "developer": sns(tds[3].text), + "expected_completion": sns(tds[4].text), + } + results.append(cert) + return results + + @staticmethod + def get_netherlands_certified(): + soup = CCSchemeDataset._download_page(constants.CC_NETHERLANDS_CERTIFIED_URL) + main_div = soup.select("body > main > div > div > div > div:nth-child(2) > div.col-lg-9 > div:nth-child(3)")[0] + rows = main_div.find_all("div", class_="row", recursive=False) + modals = main_div.find_all("div", class_="modal", recursive=False) + results = [] + for row, modal in zip(rows, modals): + row_entries = row.find_all("a") + modal_trs = modal.find_all("tr") + cert = { + "manufacturer": sns(row_entries[0].text), + "product": sns(row_entries[1].text), + "scheme": sns(row_entries[2].text), + "cert_id": sns(row_entries[3].text), + } + for tr in modal_trs: + th_text = tr.find("th").text + td = tr.find("td") + if "Manufacturer website" in th_text: + cert["manufacturer_link"] = td.find("a")["href"] + elif "Assurancelevel" in th_text: + cert["level"] = sns(td.text) + elif "Certificate" in th_text: + cert["cert_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] + elif "Certificationreport" in th_text: + cert["report_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] + elif "Securitytarget" in th_text: + cert["target_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] + elif "Maintenance report" in th_text: + cert["maintenance_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"] + results.append(cert) + return results + + @staticmethod + def get_netherlands_in_evaluation(): + soup = CCSchemeDataset._download_page(constants.CC_NETHERLANDS_INEVAL_URL) + table = soup.find("table") + results = [] + for tr in table.find_all("tr")[1:]: + tds = tr.find_all("td") + cert = { + "developer": sns(tds[0].text), + "product": sns(tds[1].text), + "category": sns(tds[2].text), + "level": sns(tds[3].text), + "certification_id": sns(tds[4].text), + } + results.append(cert) + return results + + @staticmethod + def _get_norway(url): + # TODO: Information could be expanded by following product link. + soup = CCSchemeDataset._download_page(url) + results = [] + for tr in soup.find_all("tr", class_="certified-product"): + tds = tr.find_all("td") + cert = { + "product": sns(tds[0].text), + "product_link": tds[0].find("a")["href"], + "category": sns(tds[1].find("p", class_="value").text), + "developer": sns(tds[2].find("p", class_="value").text), + "certification_date": sns(tds[3].find("time").text), + } + results.append(cert) + return results + + @staticmethod + def get_norway_certified(): + return CCSchemeDataset._get_norway(constants.CC_NORWAY_CERTIFIED_URL) + + @staticmethod + def get_norway_archived(): + return CCSchemeDataset._get_norway(constants.CC_NORWAY_ARCHIVED_URL) + + @staticmethod + def _get_korea(product_class): + # TODO: Information could be expanded by following product link. + session = requests.session() + session.get(constants.CC_KOREA_EN_URL) + # Get base page + url = constants.CC_KOREA_CERTIFIED_URL + f"?product_class={product_class}" + soup = CCSchemeDataset._download_page(url, session=session) + seen_pages = set() + pages = {1} + results = [] + while pages: + page = pages.pop() + csrf = soup.find("form", id="fm").find("input", attrs={"name": "csrf"})["value"] + resp = session.post(url, data={"csrf": csrf, "selectPage": page, "product_class": product_class}) + soup = BeautifulSoup(resp.content, "html5lib") + tbody = soup.find("table", class_="cpl").find("tbody") + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + if len(tds) != 6: + continue + link = tds[0].find("a") + id = link["id"].split("-")[1] + cert = { + "product": sns(tds[0].text), + "cert_id": sns(tds[1].text), + "product_link": constants.CC_KOREA_PRODUCT_URL.format(id), + "vendor": sns(tds[2].text), + "level": sns(tds[3].text), + "category": sns(tds[4].text), + "certification_date": sns(tds[5].text), + } + results.append(cert) + seen_pages.add(page) + page_links = soup.find("div", class_="paginate").find_all("a", class_="number_off") + for page_link in page_links: + try: + new_page = int(page_link.text) + if new_page not in seen_pages: + pages.add(new_page) + except Exception: + pass + return results + + @staticmethod + def get_korea_certified(): + return CCSchemeDataset._get_korea(product_class=1) + + @staticmethod + def get_korea_suspended(): + return CCSchemeDataset._get_korea(product_class=2) + + @staticmethod + def get_korea_archived(): + return CCSchemeDataset._get_korea(product_class=4) + + @staticmethod + def _get_singapore(url): + soup = CCSchemeDataset._download_page(url) + table = soup.find("table") + skip = False + results = [] + category_name = None + for tr in table.find_all("tr"): + if skip: + skip = False + continue + tds = tr.find_all("td") + if len(tds) == 1: + category_name = sns(tds[0].text) + skip = True + continue + + cert = { + "product": sns(tds[0].text.split()[0]), + "vendor": sns(tds[1].text), + "level": sns(tds[2].text), + "certification_date": sns(tds[3].text), + "expiration_date": sns(tds[4].text), + "category": category_name, + } + for link in tds[0].find_all("a"): + link_text = sns(link.text) + if link_text == "Certificate": + cert["cert_link"] = constants.CC_SINGAPORE_BASE_URL + link["href"] + elif link_text in ("Certificate Report", "Certification Report"): + cert["report_link"] = constants.CC_SINGAPORE_BASE_URL + link["href"] + elif link_text == "Security Target": + cert["target_link"] = constants.CC_SINGAPORE_BASE_URL + link["href"] + results.append(cert) + return results + + @staticmethod + def get_singapore_certified(): + return CCSchemeDataset._get_singapore(constants.CC_SINGAPORE_CERTIFIED_URL) + + @staticmethod + def get_singapore_in_evaluation(): + soup = CCSchemeDataset._download_page(constants.CC_SINGAPORE_CERTIFIED_URL) + header = soup.find(lambda x: x.name == "h3" and x.text == "In Evaluation") + table = header.find_next("table") + results = [] + for tr in table.find_all("tr")[1:]: + tds = tr.find_all("td") + cert = { + "name": sns(tds[0].text), + "vendor": sns(tds[1].text), + "level": sns(tds[2].text), + } + results.append(cert) + return results + + @staticmethod + def get_singapore_archived(): + return CCSchemeDataset._get_singapore(constants.CC_SINGAPORE_ARCHIVED_URL) + + @staticmethod + def get_spain_certified(): + soup = CCSchemeDataset._download_page(constants.CC_SPAIN_CERTIFIED_URL) + tbody = soup.find("table", class_="djc_items_table").find("tbody") + results = [] + for tr in tbody.find_all("tr", recursive=False): + tds = tr.find_all("td") + cert = { + "product": sns(tds[0].text), + "product_link": constants.CC_SPAIN_BASE_URL + tds[0].find("a")["href"], + "category": sns(tds[1].text), + "manufacturer": sns(tds[2].text), + "certification_date": sns(tds[3].find("td", class_="djc_value").text), + } + results.append(cert) + return results + + @staticmethod + def _get_sweden(url): + # TODO: Information could be expanded by following product link. + soup = CCSchemeDataset._download_page(url) + nav = soup.find("main").find("nav", class_="component-nav-box__list") + results = [] + for link in nav.find_all("a"): + cert = {"product": sns(link.text), "product_link": constants.CC_SWEDEN_BASE_URL + link["href"]} + results.append(cert) + return results + + @staticmethod + def get_sweden_certified(): + return CCSchemeDataset._get_sweden(constants.CC_SWEDEN_CERTIFIED_URL) + + @staticmethod + def get_sweden_in_evaluation(): + return CCSchemeDataset._get_sweden(constants.CC_SWEDEN_INEVAL_URL) + + @staticmethod + def get_sweden_archived(): + return CCSchemeDataset._get_sweden(constants.CC_SWEDEN_ARCHIVED_URL) + + @staticmethod + def get_turkey_certified(): + results = [] + with tempfile.TemporaryDirectory() as tmpdir: + pdf_path = Path(tmpdir) / "turkey.pdf" + resp = requests.get(constants.CC_TURKEY_ARCHIVED_URL) + if resp.status_code != requests.codes.ok: + raise ValueError(f"Unable to download: status={resp.status_code}") + with pdf_path.open("wb") as f: + f.write(resp.content) + dfs = tabula.read_pdf(str(pdf_path), pages="all") + for df in dfs: + for line in df.values: + cert = { + # TODO: Split item number and generate several dicts for a range they include. + "item_no": line[0], + "developer": line[1], + "product": line[2], + "cc_version": line[3], + "level": line[4], + "cert_lab": line[5], + "certification_date": line[6], + "expiration_date": line[7], + # TODO: Parse "Ongoing Evaluation" out of this field as well. + "archived": isinstance(line[9], str) and "Archived" in line[9], + } + results.append(cert) + return results + + @staticmethod + def get_usa_certified(): + # TODO: Information could be expanded by following product link. + # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). + soup = CCSchemeDataset._download_page(constants.CC_USA_CERTIFIED_URL) + tbody = soup.find("table", class_="tablesorter").find("tbody") + results = [] + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + vendor_span = tds[0].find("span", class_="b u") + product_link = tds[0].find("a") + scheme_img = tds[6].find("img") + # Only return the US certifications. + if scheme_img["title"] != "USA": + continue + cert = { + "product": sns(product_link.text), + "vendor": sns(vendor_span.text), + "product_link": product_link["href"], + "id": sns(tds[1].text), + "cc_claim": sns(tds[2].text), + "cert_lab": sns(tds[3].text), + "certification_date": sns(tds[4].text), + "assurance_maintenance_date": sns(tds[5].text), + } + results.append(cert) + return results + + @staticmethod + def get_usa_in_evaluation(): + # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). + soup = CCSchemeDataset._download_page(constants.CC_USA_INEVAL_URL) + tbody = soup.find("table", class_="tablesorter").find("tbody") + results = [] + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + vendor_span = tds[0].find("span", class_="b u") + product_name = None + for child in tds[0].children: + if isinstance(child, NavigableString): + product_name = sns(child) + break + cert = { + "vendor": sns(vendor_span.text), + "id": sns(tds[1].text), + "cc_claim": sns(tds[2].text), + "cert_lab": sns(tds[3].text), + "kickoff_date": sns(tds[4].text), + } + if product_name: + cert["product"] = product_name + results.append(cert) + return results + + @staticmethod + def get_usa_archived(): + # TODO: Information could be expanded by following the cc_claims (has links to protection profiles). + soup = CCSchemeDataset._download_page(constants.CC_USA_ARCHIVED_URL) + tbody = soup.find("table", class_="tablesorter").find("tbody") + results = [] + for tr in tbody.find_all("tr"): + tds = tr.find_all("td") + scheme_img = tds[5].find("img") + # Only return the US certifications. + if scheme_img["title"] != "USA": + continue + vendor_span = tds[0].find("span", class_="b u") + product_name = None + for child in tds[0].children: + if isinstance(child, NavigableString): + product_name = sns(child) + break + cert = { + "vendor": sns(vendor_span.text), + "id": sns(tds[1].text), + "cc_claim": sns(tds[2].text), + "cert_lab": sns(tds[3].text), + "certification_date": sns(tds[4].text), + } + if product_name: + cert["product"] = product_name + results.append(cert) + return results diff --git a/src/sec_certs/dataset/cpe.py b/src/sec_certs/dataset/cpe.py new file mode 100644 index 00000000..927ce674 --- /dev/null +++ b/src/sec_certs/dataset/cpe.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import copy +import itertools +import logging +import tempfile +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path +from typing import ClassVar, Iterator + +import pandas as pd + +from sec_certs import constants +from sec_certs.dataset.cve import CVEDataset +from sec_certs.dataset.json_path_dataset import JSONPathDataset +from sec_certs.sample.cpe import CPE, cached_cpe +from sec_certs.serialization.json import ComplexSerializableType, serialize +from sec_certs.utils import helpers +from sec_certs.utils.tqdm import tqdm + +logger = logging.getLogger(__name__) + + +class CPEDataset(JSONPathDataset, ComplexSerializableType): + """ + Dataset of CPE records. Includes look-up dictionaries for fast search. + """ + + CPE_XML_BASENAME: ClassVar[str] = "official-cpe-dictionary_v2.3.xml" + CPE_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/xml/cpe/dictionary/" + CPE_XML_BASENAME + ".zip" + + def __init__( + self, + was_enhanced_with_vuln_cpes: bool, + cpes: dict[str, CPE], + json_path: str | Path = constants.DUMMY_NONEXISTING_PATH, + ): + self.was_enhanced_with_vuln_cpes = was_enhanced_with_vuln_cpes + self.cpes = cpes + self.json_path = Path(json_path) + + self.vendor_to_versions: dict[str, set[str]] = dict() + self.vendor_version_to_cpe: dict[tuple[str, str], set[CPE]] = dict() + self.title_to_cpes: dict[str, set[CPE]] = dict() + self.vendors: set[str] = set() + + self.build_lookup_dicts() + + def __iter__(self) -> Iterator[CPE]: + yield from self.cpes.values() + + def __getitem__(self, item: str) -> CPE: + return self.cpes.__getitem__(item.lower()) + + def __setitem__(self, key: str, value: CPE) -> None: + self.cpes.__setitem__(key.lower(), value) + + def __len__(self) -> int: + return len(self.cpes) + + def __contains__(self, item: CPE) -> bool: + if not isinstance(item, CPE): + raise ValueError(f"{item} is not of CPE class") + return item.uri in self.cpes.keys() and self.cpes[item.uri] == item + + def __eq__(self, other: object) -> bool: + return isinstance(other, CPEDataset) and self.cpes == other.cpes + + @property + def serialized_attributes(self) -> list[str]: + return ["was_enhanced_with_vuln_cpes", "cpes"] + + def build_lookup_dicts(self) -> None: + """ + Will build look-up dictionaries that are used for fast matching. + """ + logger.info("CPE dataset: building lookup dictionaries.") + self.vendor_to_versions = {x.vendor: set() for x in self} + self.vendor_version_to_cpe = dict() + self.title_to_cpes = dict() + self.vendors = set(self.vendor_to_versions.keys()) + for cpe in self: + self.vendor_to_versions[cpe.vendor].add(cpe.version) + if (cpe.vendor, cpe.version) not in self.vendor_version_to_cpe: + self.vendor_version_to_cpe[(cpe.vendor, cpe.version)] = {cpe} + else: + self.vendor_version_to_cpe[(cpe.vendor, cpe.version)].add(cpe) + + if cpe.title: + if cpe.title not in self.title_to_cpes: + self.title_to_cpes[cpe.title] = {cpe} + else: + self.title_to_cpes[cpe.title].add(cpe) + + @classmethod + def from_web(cls, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> CPEDataset: + """ + Creates CPEDataset from NIST resources published on-line + + :param Union[str, Path] json_path: Path to store the dataset to + :return CPEDataset: The resulting dataset + """ + with tempfile.TemporaryDirectory() as tmp_dir: + xml_path = Path(tmp_dir) / cls.CPE_XML_BASENAME + zip_path = Path(tmp_dir) / (cls.CPE_XML_BASENAME + ".zip") + helpers.download_file(cls.CPE_URL, zip_path) + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(tmp_dir) + + return cls._from_xml(xml_path, json_path) + + @classmethod + def _from_xml(cls, xml_path: str | Path, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> CPEDataset: + logger.info("Loading CPE dataset from XML.") + root = ET.parse(xml_path).getroot() + dct = {} + for cpe_item in root.findall("{http://cpe.mitre.org/dictionary/2.0}cpe-item"): + found_title = cpe_item.find("{http://cpe.mitre.org/dictionary/2.0}title") + if found_title is None: + raise RuntimeError( + "Title is not found during building CPE dataset from xml - this should not be happening" + ) + title = found_title.text + + found_cpe_uri = cpe_item.find("{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item") + if found_cpe_uri is None: + raise RuntimeError( + "CPE uri is not found during building CPE dataset from xml - this should not be happening" + ) + cpe_uri = found_cpe_uri.attrib["name"] + + dct[cpe_uri] = cached_cpe(cpe_uri, title) + + return cls(False, dct, json_path) + + def to_pandas(self) -> pd.DataFrame: + """ + Turns the dataset into pandas DataFrame. Each CPE record forms a row. + + :return pd.DataFrame: the resulting DataFrame + """ + df = pd.DataFrame([x.pandas_tuple for x in self], columns=CPE.pandas_columns) + df = df.set_index("uri") + return df + + @serialize + def enhance_with_cpes_from_cve_dataset(self, cve_dset: CVEDataset | str | Path) -> None: + """ + Some CPEs are present only in the CVEDataset and are missing from the CPE Dataset. + This method goes through the provided CVEDataset and enriches self with CPEs from + the CVEDataset. + + :param Union[CVEDataset, str, Path] cve_dset: CVEDataset of a path to it. + """ + + def _adding_condition( + considered_cpe: CPE, + vndr_item_lookup: set[tuple[str, str]], + vndr_item_version_lookup: set[tuple[str, str, str]], + ) -> bool: + if ( + considered_cpe.version == constants.CPE_VERSION_NA + and (considered_cpe.vendor, considered_cpe.item_name) not in vndr_item_lookup + ): + return True + elif ( + considered_cpe.version != constants.CPE_VERSION_NA + and (considered_cpe.vendor, considered_cpe.item_name, considered_cpe.version) + not in vndr_item_version_lookup + ): + return True + return False + + if isinstance(cve_dset, (str, Path)): + cve_dset = CVEDataset.from_json(cve_dset) + + if not isinstance(cve_dset, CVEDataset): + raise RuntimeError("Conversion of CVE dataset did not work.") + all_cpes_in_cve_dset = set(itertools.chain.from_iterable(cve.vulnerable_cpes for cve in cve_dset)) + + old_len = len(self.cpes) + + # We only enrich if tuple (vendor, item_name) is not already in the dataset + vendor_item_lookup = {(cpe.vendor, cpe.item_name) for cpe in self} + vendor_item_version_lookup = {(cpe.vendor, cpe.item_name, cpe.version) for cpe in self} + for cpe in tqdm(all_cpes_in_cve_dset, desc="Enriching CPE dataset with new CPEs"): + if _adding_condition(cpe, vendor_item_lookup, vendor_item_version_lookup): + new_cpe = copy.deepcopy(cpe) + new_cpe.start_version = None + new_cpe.end_version = None + self[new_cpe.uri] = new_cpe + self.build_lookup_dicts() + + logger.info(f"Enriched the CPE dataset with {len(self.cpes) - old_len} new CPE records.") + self.was_enhanced_with_vuln_cpes = True diff --git a/src/sec_certs/dataset/cve.py b/src/sec_certs/dataset/cve.py new file mode 100644 index 00000000..ebbd9171 --- /dev/null +++ b/src/sec_certs/dataset/cve.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import datetime +import glob +import itertools +import json +import logging +import shutil +import tempfile +import zipfile +from pathlib import Path +from typing import ClassVar + +import numpy as np +import pandas as pd + +import sec_certs.constants as constants +import sec_certs.utils.helpers as helpers +from sec_certs.config.configuration import config +from sec_certs.dataset.json_path_dataset import JSONPathDataset +from sec_certs.sample.cpe import CPE, cached_cpe +from sec_certs.sample.cve import CVE +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.utils.parallel_processing import process_parallel +from sec_certs.utils.tqdm import tqdm + +logger = logging.getLogger(__name__) + + +class CVEDataset(JSONPathDataset, ComplexSerializableType): + CVE_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" + CPE_MATCH_FEED_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip" + + def __init__(self, cves: dict[str, CVE], json_path: str | Path = constants.DUMMY_NONEXISTING_PATH): + self.cves = cves + self.json_path = Path(json_path) + self.cpe_to_cve_ids_lookup: dict[str, set[str]] = dict() + + @property + def serialized_attributes(self) -> list[str]: + return ["cves"] + + def __iter__(self): + yield from self.cves.values() + + def __getitem__(self, item: str) -> CVE: + return self.cves.__getitem__(item.upper()) + + def __setitem__(self, key: str, value: CVE): + self.cves.__setitem__(key.lower(), value) + + def __len__(self) -> int: + return len(self.cves) + + def __eq__(self, other: object): + return isinstance(other, CVEDataset) and self.cves == other.cves + + def build_lookup_dict(self, use_nist_mapping: bool = True, nist_matching_filepath: Path | None = None): + """ + Builds look-up dictionary CPE -> Set[CVE] + Developer's note: There are 3 CPEs that are present in the cpe matching feed, but are badly processed by CVE + feed, in which case they won't be found as a key in the dictionary. We intentionally ignore those. Feel free + to add corner cases and manual fixes. According to our investigation, the suffereing CPEs are: + - CPE(uri='cpe:2.3:a:arubanetworks:airwave:*:*:*:*:*:*:*:*', title=None, version='*', vendor='arubanetworks', item_name='airwave', start_version=None, end_version=('excluding', '8.2.0.0')) + - CPE(uri='cpe:2.3:a:bayashi:dopvcomet\\*:0009:b:*:*:*:*:*:*', title=None, version='0009', vendor='bayashi', item_name='dopvcomet\\*', start_version=None, end_version=None) + - CPE(uri='cpe:2.3:a:bayashi:dopvstar\\*:0091:*:*:*:*:*:*:*', title=None, version='0091', vendor='bayashi', item_name='dopvstar\\*', start_version=None, end_version=None) + """ + self.cpe_to_cve_ids_lookup = dict() + self.cves = {x.cve_id.upper(): x for x in self} + + logger.info("Getting CPE matching dictionary from NIST.gov") + + if use_nist_mapping: + matching_dict = self.get_nist_cpe_matching_dict(nist_matching_filepath) + + cve: CVE + for cve in tqdm(self, desc="Building-up lookup dictionaries for fast CVE matching"): + # See note above, we use matching_dict.get(cpe, []) instead of matching_dict[cpe] as would be expected + if use_nist_mapping: + vulnerable_configurations = list( + itertools.chain.from_iterable(matching_dict.get(cpe, []) for cpe in cve.vulnerable_cpes) + ) + else: + vulnerable_configurations = cve.vulnerable_cpes + for cpe in vulnerable_configurations: + if cpe.uri not in self.cpe_to_cve_ids_lookup: + self.cpe_to_cve_ids_lookup[cpe.uri] = {cve.cve_id} + else: + self.cpe_to_cve_ids_lookup[cpe.uri].add(cve.cve_id) + + @classmethod + def download_cves(cls, output_path_str: str, start_year: int, end_year: int): + output_path = Path(output_path_str) + if not output_path.exists(): + output_path.mkdir() + + urls = [cls.CVE_URL + str(x) + ".json.zip" for x in range(start_year, end_year + 1)] + + logger.info(f"Identified {len(urls)} CVE files to fetch from nist.gov. Downloading them into {output_path}") + with tempfile.TemporaryDirectory() as tmp_dir: + outpaths = [Path(tmp_dir) / Path(x).name.rstrip(".zip") for x in urls] + responses = helpers.download_parallel(urls, outpaths, "Downloading CVEs resources from NVD") + + for o, r in zip(outpaths, responses): + if r == constants.RESPONSE_OK: + with zipfile.ZipFile(o, "r") as zip_handle: + zip_handle.extractall(output_path) + + @classmethod + def from_nist_json(cls, input_path: str) -> CVEDataset: + with Path(input_path).open("r") as handle: + data = json.load(handle) + cves = [CVE.from_nist_dict(x) for x in data["CVE_Items"]] + return cls({x.cve_id: x for x in cves}) + + @classmethod + def from_web( + cls, + start_year: int = 2002, + end_year: int = datetime.datetime.now().year, + json_path: str | Path = constants.DUMMY_NONEXISTING_PATH, + ): + logger.info("Building CVE dataset from nist.gov website.") + with tempfile.TemporaryDirectory() as tmp_dir: + cls.download_cves(tmp_dir, start_year, end_year) + json_files = glob.glob(tmp_dir + "/*.json") + + all_cves = dict() + logger.info("Downloaded required resources. Building CVEDataset from jsons.") + results = process_parallel( + cls.from_nist_json, + json_files, + config.n_threads, + use_threading=False, + progress_bar_desc="Building CVEDataset from jsons", + ) + for r in results: + all_cves.update(r.cves) + + return cls(all_cves, json_path) + + def get_cve_ids_for_cpe_uri(self, cpe_uri: str) -> set[str] | None: + return self.cpe_to_cve_ids_lookup.get(cpe_uri, None) + + def filter_related_cpes(self, relevant_cpes: set[CPE]): + """ + Since each of the CVEs is related to many CPEs, the dataset size explodes (serialized). For certificates, + only CPEs within sample dataset are relevant. This function modifies all CVE elements. Specifically, it + deletes all CPE records unless they are part of relevant_cpe_uris. + :param relevant_cpes: List of relevant CPEs to keep in CVE dataset. + """ + total_deleted_cpes = 0 + cve_ids_to_delete = [] + for cve in self: + n_cpes_orig = len(cve.vulnerable_cpes) + cve.vulnerable_cpes = list(filter(lambda x: x in relevant_cpes, cve.vulnerable_cpes)) + total_deleted_cpes += n_cpes_orig - len(cve.vulnerable_cpes) + if not cve.vulnerable_cpes: + cve_ids_to_delete.append(cve.cve_id) + + for cve_id in cve_ids_to_delete: + del self.cves[cve_id] + logger.info( + f"Totally deleted {total_deleted_cpes} irrelevant CPEs and {len(cve_ids_to_delete)} CVEs from CVEDataset." + ) + + def to_pandas(self) -> pd.DataFrame: + df = pd.DataFrame([x.pandas_tuple for x in self], columns=CVE.pandas_columns) + df.cwe_ids = df.cwe_ids.map(lambda x: x if x else np.nan) + return df.set_index("cve_id") + + def get_nist_cpe_matching_dict(self, input_filepath: Path | None) -> dict[CPE, list[CPE]]: + """ + Computes dictionary that maps complex CPEs to list of simple CPEs. + """ + + def parse_key_cpe(field: dict) -> CPE: + start_version = None + if "versionStartIncluding" in field: + start_version = ("including", field["versionStartIncluding"]) + elif "versionStartExcluding" in field: + start_version = ("excluding", field["versionStartExcluding"]) + + end_version = None + if "versionEndIncluding" in field: + end_version = ("including", field["versionEndIncluding"]) + elif "versionEndExcluding" in field: + end_version = ("excluding", field["versionEndExcluding"]) + + return cached_cpe(field["cpe23Uri"], start_version=start_version, end_version=end_version) + + def parse_values_cpe(field: dict) -> list[CPE]: + return [cached_cpe(x["cpe23Uri"]) for x in field["cpe_name"]] + + logger.debug("Attempting to get NIST mapping file.") + if not input_filepath or not input_filepath.is_file(): + logger.debug("NIST mapping file not available, going to download.") + with tempfile.TemporaryDirectory() as tmp_dir: + filename = Path(self.CPE_MATCH_FEED_URL).name + download_path = Path(tmp_dir) / filename + unzipped_path = Path(tmp_dir) / filename.rstrip(".zip") + helpers.download_file(self.CPE_MATCH_FEED_URL, download_path) + + with zipfile.ZipFile(download_path, "r") as zip_handle: + zip_handle.extractall(tmp_dir) + with unzipped_path.open("r") as handle: + match_data = json.load(handle) + if input_filepath: + logger.debug(f"Copying attained NIST mapping file to {input_filepath}") + shutil.move(str(unzipped_path), str(input_filepath)) + else: + with input_filepath.open("r") as handle: + match_data = json.load(handle) + + mapping_dict = dict() + for match in tqdm(match_data["matches"], desc="parsing cpe matching (by NIST) dictionary"): + key = parse_key_cpe(match) + value = parse_values_cpe(match) + mapping_dict[key] = value if value else [key] + + return mapping_dict diff --git a/src/sec_certs/dataset/dataset.py b/src/sec_certs/dataset/dataset.py new file mode 100644 index 00000000..1c89e771 --- /dev/null +++ b/src/sec_certs/dataset/dataset.py @@ -0,0 +1,572 @@ +from __future__ import annotations + +import itertools +import json +import logging +import re +import shutil +import tempfile +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Generic, Iterator, TypeVar, cast + +import pandas as pd + +import sec_certs.constants as constants +import sec_certs.utils.helpers as helpers +from sec_certs.config.configuration import config +from sec_certs.dataset.cpe import CPEDataset +from sec_certs.dataset.cve import CVEDataset +from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.sample.certificate import Certificate +from sec_certs.sample.cpe import CPE +from sec_certs.serialization.json import ComplexSerializableType, get_class_fullname, serialize +from sec_certs.utils.tqdm import tqdm + +logger = logging.getLogger(__name__) + + +@dataclass +class AuxillaryDatasets: + cpe_dset: CPEDataset | None = None + cve_dset: CVEDataset | None = None + + +CertSubType = TypeVar("CertSubType", bound=Certificate) +AuxillaryDatasetsSubType = TypeVar("AuxillaryDatasetsSubType", bound=AuxillaryDatasets) +DatasetSubType = TypeVar("DatasetSubType", bound="Dataset") + + +class Dataset(Generic[CertSubType, AuxillaryDatasetsSubType], ComplexSerializableType, ABC): + """ + Base class for dataset of certificates from CC and FIPS 140 schemes. Layouts public + functions, the processing pipeline and common operations on the dataset and certs. + """ + + @dataclass + class DatasetInternalState(ComplexSerializableType): + meta_sources_parsed: bool = False + artifacts_downloaded: bool = False + pdfs_converted: bool = False + auxillary_datasets_processed: bool = False + certs_analyzed: bool = False + + def __bool__(self): + return any(vars(self)) + + def __init__( + self, + certs: dict[str, CertSubType] = dict(), + root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH, + name: str | None = None, + description: str = "", + state: DatasetInternalState | None = None, + auxillary_datasets: AuxillaryDatasetsSubType | None = None, + ): + self.certs = certs + + self.timestamp = datetime.now() + self.sha256_digest = "not implemented" + self.name = name if name else type(self).__name__.lower() + "_dataset" + self.description = description if description else "No description provided" + self.state = state if state else self.DatasetInternalState() + + if not auxillary_datasets: + self.auxillary_datasets = AuxillaryDatasets() + else: + self.auxillary_datasets = auxillary_datasets + + self.root_dir = Path(root_dir) + + @property + def root_dir(self) -> Path: + """ + Directory that will hold the serialized dataset files. + """ + return self._root_dir + + @root_dir.setter + def root_dir(self: DatasetSubType, new_dir: str | Path) -> None: + """ + This setter will only set the root dir and all internal paths so that they point + to the new root dir. No data is being moved around. + """ + new_dir = Path(new_dir) + if new_dir.is_file(): + raise ValueError(f"Root dir of {get_class_fullname(self)} cannot be a file.") + + self._root_dir = new_dir + self._set_local_paths() + + @property + def web_dir(self) -> Path: + """ + Path to certification-artifacts posted on web. + """ + return self.root_dir / "web" + + @property + def auxillary_datasets_dir(self) -> Path: + """ + Path to directory with auxillary datasets. + """ + return self.root_dir / "auxillary_datasets" + + @property + def certs_dir(self) -> Path: + """ + Returns directory that holds files associated with certificates + """ + return self.root_dir / "certs" + + @property + def cpe_dataset_path(self) -> Path: + return self.auxillary_datasets_dir / "cpe_dataset.json" + + @property + def cve_dataset_path(self) -> Path: + return self.auxillary_datasets_dir / "cve_dataset.json" + + @property + def nist_cve_cpe_matching_dset_path(self) -> Path: + return self.auxillary_datasets_dir / "nvdcpematch-1.0.json" + + @property + def json_path(self) -> Path: + return self.root_dir / (self.name + ".json") + + def __contains__(self, item: object) -> bool: + if not isinstance(item, Certificate): + raise TypeError( + f"You attempted to check if {type(item)} is member of {type(self)}, but only {type(Certificate)} are allowed to be members." + ) + return item.dgst in self.certs + + def __iter__(self) -> Iterator[CertSubType]: + yield from self.certs.values() + + def __getitem__(self, item: str) -> CertSubType: + return self.certs.__getitem__(item.lower()) + + def __setitem__(self, key: str, value: CertSubType): + self.certs.__setitem__(key.lower(), value) + + def __len__(self) -> int: + return len(self.certs) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Dataset): + return NotImplemented + return self.certs == other.certs + + def __str__(self) -> str: + return str(type(self).__name__) + ":" + self.name + ", " + str(len(self)) + " certificates" + + @classmethod + def from_web(cls: type[DatasetSubType], url: str, progress_bar_desc: str, filename: str) -> DatasetSubType: + """ + Fetches a fully processed dataset instance from static site that hosts it. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + dset_path = Path(tmp_dir) / filename + helpers.download_file(url, dset_path, show_progress_bar=True, progress_bar_desc=progress_bar_desc) + dset = cls.from_json(dset_path) + dset.root_dir = constants.DUMMY_NONEXISTING_PATH + return dset + + def to_dict(self) -> dict[str, Any]: + return { + "state": self.state, + "timestamp": self.timestamp, + "sha256_digest": self.sha256_digest, + "name": self.name, + "description": self.description, + "n_certs": len(self), + "certs": list(self.certs.values()), + } + + @classmethod + def from_dict(cls: type[DatasetSubType], dct: dict) -> DatasetSubType: + certs = {x.dgst: x for x in dct["certs"]} + dset = cls(certs, name=dct["name"], description=dct["description"], state=dct["state"]) + if len(dset) != (claimed := dct["n_certs"]): + logger.error( + f"The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed})." + ) + return dset + + @classmethod + def from_json(cls: type[DatasetSubType], input_path: str | Path) -> DatasetSubType: + dset = cast("DatasetSubType", ComplexSerializableType.from_json(input_path)) + dset._root_dir = Path(input_path).parent.absolute() + dset._set_local_paths() + return dset + + def _set_local_paths(self) -> None: + if self.auxillary_datasets.cpe_dset: + self.auxillary_datasets.cpe_dset.json_path = self.cpe_dataset_path + if self.auxillary_datasets.cve_dset: + self.auxillary_datasets.cve_dset.json_path = self.cve_dataset_path + + def move_dataset(self, new_root_dir: str | Path) -> None: + """ + Moves all dataset files to `new_root_dir` and adjusts all paths internally. Deletes the artifacts from the original location. + :param str | Path new_root_dir: path to directory where the new dataset shall be stored. + """ + new_root_dir = Path(new_root_dir) + if new_root_dir.is_file(): + raise ValueError("New root dir must be a directory, not an existing file.") + new_root_dir.mkdir(parents=True, exist_ok=True) + + shutil.copytree(str(self.root_dir), str(new_root_dir), dirs_exist_ok=True) + shutil.rmtree(self.root_dir) + self.root_dir = new_root_dir + + def copy_dataset(self, new_root_dir: str | Path) -> None: + """ + Copies all dataset files to `new_root_dir` and adjusts all paths internally. Keeps the artifacts from the original location. + :param str | Path new_root_dir: path to directory where the new dataset shall be stored. + """ + new_root_dir = Path(new_root_dir) + if new_root_dir.is_file(): + raise ValueError("New root dir must be a directory, not an existing file.") + new_root_dir.mkdir(parents=True, exist_ok=True) + + shutil.copytree(str(self.root_dir), str(new_root_dir), dirs_exist_ok=True) + self.root_dir = new_root_dir + + def _get_certs_by_name(self, name: str) -> set[CertSubType]: + """ + Returns list of certificates that match given name. + """ + return {crt for crt in self if crt.name and crt.name == name} + + @abstractmethod + def get_certs_from_web(self) -> None: + raise NotImplementedError("Not meant to be implemented by the base class.") + + @serialize + @abstractmethod + def process_auxillary_datasets(self, download_fresh: bool = False) -> None: + """ + Processes all auxillary datasets (CPE, CVE, ...) that are required during computation. + """ + logger.info("Processing auxillary datasets.") + self.auxillary_datasets_dir.mkdir(parents=True, exist_ok=True) + self.auxillary_datasets.cpe_dset = self._prepare_cpe_dataset(download_fresh) + self.auxillary_datasets.cve_dset = self._prepare_cve_dataset(download_fresh_cves=download_fresh) + self.state.auxillary_datasets_processed = True + + @serialize + def download_all_artifacts(self, fresh: bool = True) -> None: + """ + Downloads all artifacts related to certification in the given scheme. + """ + if not self.state.meta_sources_parsed: + logger.error("Attempting to download pdfs while not having csv/html meta-sources parsed. Returning.") + return + + logger.info("Attempting to download certification artifacts.") + self._download_all_artifacts_body(fresh) + if fresh: + self._download_all_artifacts_body(False) + + self.state.artifacts_downloaded = True + + @abstractmethod + def _download_all_artifacts_body(self, fresh: bool = True) -> None: + raise NotImplementedError("Not meant to be implemented by the base class.") + + @serialize + def convert_all_pdfs(self, fresh: bool = True) -> None: + """ + Converts all pdf artifacts to txt, given the certification scheme. + """ + if not self.state.artifacts_downloaded: + logger.error("Attempting to convert pdfs while not having the artifacts downloaded. Returning.") + return + + logger.info("Converting all PDFs to txt") + self._convert_all_pdfs_body(fresh) + if fresh: + self._convert_all_pdfs_body(False) + + self.state.pdfs_converted = True + + @abstractmethod + def _convert_all_pdfs_body(self, fresh: bool = True) -> None: + raise NotImplementedError("Not meant to be implemented by the base class.") + + @serialize + def analyze_certificates(self) -> None: + """ + Does two things: + - Extracts data from certificates (keywords, etc.) + - Computes various heuristics on the certificates. + """ + if not self.state.pdfs_converted: + logger.info( + "Attempting run analysis of txt files while not having the pdf->txt conversion done. Returning." + ) + return + if not self.state.auxillary_datasets_processed: + logger.info( + "Attempting to run analysis of certifies while not having the auxillary datasets processed. Returning." + ) + + logger.info("Analyzing certificates.") + self._analyze_certificates_body() + self.state.certs_analyzed = True + + def _analyze_certificates_body(self) -> None: + self.extract_data() + self._compute_heuristics() + + @abstractmethod + def extract_data(self) -> None: + raise NotImplementedError("Not meant to be implemented by the base class.") + + def _compute_heuristics(self) -> None: + logger.info("Computing various heuristics from the certificates.") + self.compute_cpe_heuristics() + self.compute_related_cves() + self._compute_references() + self._compute_transitive_vulnerabilities() + + @abstractmethod + def _compute_references(self) -> None: + raise NotImplementedError("Not meant to be implemented by the base class.") + + @abstractmethod + def _compute_transitive_vulnerabilities(self) -> None: + raise NotImplementedError("Not meant to be implemented by the base class.") + + def _prepare_cpe_dataset(self, download_fresh_cpes: bool = False) -> CPEDataset: + logger.info("Preparing CPE dataset.") + if not self.auxillary_datasets_dir.exists(): + self.auxillary_datasets_dir.mkdir(parents=True) + + if not self.cpe_dataset_path.exists() or download_fresh_cpes is True: + cpe_dataset = CPEDataset.from_web(self.cpe_dataset_path) + cpe_dataset.to_json() + else: + cpe_dataset = CPEDataset.from_json(str(self.cpe_dataset_path)) + + return cpe_dataset + + def _prepare_cve_dataset( + self, download_fresh_cves: bool = False, use_nist_cpe_matching_dict: bool = True + ) -> CVEDataset: + logger.info("Preparing CVE dataset.") + if not self.auxillary_datasets_dir.exists(): + self.auxillary_datasets_dir.mkdir(parents=True) + + if not self.cve_dataset_path.exists() or download_fresh_cves is True: + cve_dataset = CVEDataset.from_web(json_path=self.cve_dataset_path) + cve_dataset.to_json() + else: + cve_dataset = CVEDataset.from_json(self.cve_dataset_path) + + cve_dataset.build_lookup_dict(use_nist_cpe_matching_dict, self.nist_cve_cpe_matching_dset_path) + return cve_dataset + + @serialize + def compute_cpe_heuristics(self, download_fresh_cpes: bool = False) -> CPEClassifier: + """ + Computes matching CPEs for the certificates. + """ + WINDOWS_WEAK_CPES: set[CPE] = { + CPE("cpe:2.3:o:microsoft:windows:-:*:*:*:*:*:x64:*", "Microsoft Windows on X64", None, None), + CPE("cpe:2.3:o:microsoft:windows:-:*:*:*:*:*:x86:*", "Microsoft Windows on X86", None, None), + } + + def filter_condition(cpe: CPE) -> bool: + """ + Filters out very weak CPE matches that don't improve our database. + """ + if ( + cpe.title + and (cpe.version == "-" or cpe.version == "*") + and not any(char.isdigit() for char in cpe.title) + ): + return False + elif ( + not cpe.title + and cpe.item_name + and (cpe.version == "-" or cpe.version == "*") + and not any(char.isdigit() for char in cpe.item_name) + ): + return False + elif re.match(constants.RELEASE_CANDIDATE_REGEX, cpe.update): + return False + elif cpe in WINDOWS_WEAK_CPES: + return False + return True + + logger.info("Computing heuristics: Finding CPE matches for certificates") + if not self.auxillary_datasets.cpe_dset or download_fresh_cpes: + self.auxillary_datasets.cpe_dset = self._prepare_cpe_dataset(download_fresh_cpes) + + # Temporarily disabled, see: https://github.com/crocs-muni/sec-certs/issues/173 + # if not cpe_dset.was_enhanced_with_vuln_cpes: + # self.auxillary_datasets.cve_dset = self._prepare_cve_dataset(download_fresh_cves=False) + # self.auxillary_datasets.cpe_dset.enhance_with_cpes_from_cve_dataset(cve_dset) # this also calls build_lookup_dicts() on cpe_dset + # else: + # self.auxillary_datasets.cpe_dset.build_lookup_dicts() + + clf = CPEClassifier(config.cpe_matching_threshold, config.cpe_n_max_matches) + clf.fit([x for x in self.auxillary_datasets.cpe_dset if filter_condition(x)]) + + cert: CertSubType + for cert in tqdm(self, desc="Predicting CPE matches with the classifier"): + cert.compute_heuristics_version() + + cert.heuristics.cpe_matches = ( + clf.predict_single_cert(cert.manufacturer, cert.name, cert.heuristics.extracted_versions) + if cert.name + else None + ) + + return clf + + def to_label_studio_json(self, output_path: str | Path) -> None: + cpe_dset = self._prepare_cpe_dataset() + + lst = [] + for cert in [x for x in cast(Iterator[Certificate], self) if x.heuristics.cpe_matches]: + dct = {"text": cert.label_studio_title} + candidates = [cpe_dset[x].title for x in cert.heuristics.cpe_matches] + candidates += ["No good match"] * (config.cpe_n_max_matches - len(candidates)) + options = ["option_" + str(x) for x in range(1, config.cpe_n_max_matches)] + dct.update({o: c for o, c in zip(options, candidates)}) + lst.append(dct) + + with Path(output_path).open("w") as handle: + json.dump(lst, handle, indent=4) + + @serialize + def load_label_studio_labels(self, input_path: str | Path) -> set[str]: + with Path(input_path).open("r") as handle: + data = json.load(handle) + + cpe_dset = self._prepare_cpe_dataset() + labeled_cert_digests: set[str] = set() + + logger.info("Translating label studio matches into their CPE representations and assigning to certificates.") + for annotation in tqdm(data, desc="Translating label studio matches"): + cpe_candidate_keys = { + key for key in annotation.keys() if "option_" in key and annotation[key] != "No good match" + } + + if "verified_cpe_match" not in annotation: + incorrect_keys: set[str] = set() + elif isinstance(annotation["verified_cpe_match"], str): + incorrect_keys = {annotation["verified_cpe_match"]} + else: + incorrect_keys = set(annotation["verified_cpe_match"]["choices"]) + + incorrect_keys = {x.lstrip("$") for x in incorrect_keys} + predicted_annotations = {annotation[x] for x in cpe_candidate_keys - incorrect_keys} + + cpes: set[CPE] = set() + for x in predicted_annotations: + if x not in cpe_dset.title_to_cpes: + logger.error(f"{x} not in dataset") + else: + to_update = cpe_dset.title_to_cpes[x] + if to_update and not cpes: + cpes = to_update + elif to_update and cpes: + cpes.update(to_update) + + # distinguish between FIPS and CC + if "\n" in annotation["text"]: + cert_name = annotation["text"].split("\nModule name: ")[1].split("\n")[0] + else: + cert_name = annotation["text"] + + certs = self._get_certs_by_name(cert_name) + labeled_cert_digests.update({x.dgst for x in certs}) + + for c in certs: + c.heuristics.verified_cpe_matches = {x.uri for x in cpes if x is not None} if cpes else None + + return labeled_cert_digests + + def enrich_automated_cpes_with_manual_labels(self) -> None: + """ + Prior to CVE matching, it is wise to expand the database of automatic CPE matches with those that were manually assigned. + """ + for cert in cast(Iterator[Certificate], self): + if not cert.heuristics.cpe_matches and cert.heuristics.verified_cpe_matches: + cert.heuristics.cpe_matches = cert.heuristics.verified_cpe_matches + elif cert.heuristics.cpe_matches and cert.heuristics.verified_cpe_matches: + cert.heuristics.cpe_matches = set(cert.heuristics.cpe_matches).union( + set(cert.heuristics.verified_cpe_matches) + ) + + @serialize + def compute_related_cves( + self, + download_fresh_cves: bool = False, + use_nist_cpe_matching_dict: bool = True, + ) -> None: + """ + Computes CVEs for the certificates, given their CPE matches. + """ + logger.info("Retrieving related CVEs to verified CPE matches") + if download_fresh_cves or not self.auxillary_datasets.cve_dset: + self.auxillary_datasets.cve_dset = self._prepare_cve_dataset( + download_fresh_cves, use_nist_cpe_matching_dict + ) + + logger.info("Computing heuristics: CVEs in certificates.") + self.enrich_automated_cpes_with_manual_labels() + cpe_rich_certs = [x for x in cast(Iterator[Certificate], self) if x.heuristics.cpe_matches] + + if not cpe_rich_certs: + logger.error( + "No certificates with verified CPE match detected. You must run dset.manually_verify_cpe_matches() first. Returning." + ) + return + + relevant_cpes = set(itertools.chain.from_iterable(x.heuristics.cpe_matches for x in cpe_rich_certs)) + self.auxillary_datasets.cve_dset.filter_related_cpes(relevant_cpes) + + cert: Certificate + for cert in tqdm(cpe_rich_certs, desc="Computing related CVES"): + if cert.heuristics.cpe_matches: + related_cves = [ + self.auxillary_datasets.cve_dset.get_cve_ids_for_cpe_uri(x) for x in cert.heuristics.cpe_matches + ] + related_cves = list(filter(lambda x: x is not None, related_cves)) + if related_cves: + cert.heuristics.related_cves = set( + itertools.chain.from_iterable(x for x in related_cves if x is not None) + ) + else: + cert.heuristics.related_cves = None + + n_vulnerable = len([x for x in cpe_rich_certs if x.heuristics.related_cves]) + n_vulnerabilities = sum([len(x.heuristics.related_cves) for x in cpe_rich_certs if x.heuristics.related_cves]) + logger.info( + f"In total, we identified {n_vulnerabilities} vulnerabilities in {n_vulnerable} vulnerable certificates." + ) + + def get_keywords_df(self, var: str) -> pd.DataFrame: + """ + Get dataframe of keyword hits for attribute (var) that is member of PdfData class. + """ + data = [dict({"dgst": x.dgst}, **x.pdf_data.get_keywords_df_data(var)) for x in self] + return pd.DataFrame(data).set_index("dgst") + + def update_with_certs(self, certs: list[CertSubType]) -> None: + """ + Enriches the dataset with `certs` + :param List[CommonCriteriaCert] certs: new certs to include into the dataset. + """ + if any([x not in self for x in certs]): + logger.warning("Updating dataset with certificates outside of the dataset!") + self.certs.update({x.dgst: x for x in certs}) diff --git a/src/sec_certs/dataset/fips.py b/src/sec_certs/dataset/fips.py new file mode 100644 index 00000000..14571f93 --- /dev/null +++ b/src/sec_certs/dataset/fips.py @@ -0,0 +1,355 @@ +from __future__ import annotations + +import datetime +import itertools +import logging +import shutil +from pathlib import Path +from typing import Final + +import numpy as np +import pandas as pd +from bs4 import BeautifulSoup, NavigableString + +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.dataset.cpe import CPEDataset +from sec_certs.dataset.cve import CVEDataset +from sec_certs.dataset.dataset import AuxillaryDatasets, Dataset +from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset +from sec_certs.model.reference_finder import ReferenceFinder +from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder +from sec_certs.sample.fips import FIPSCertificate +from sec_certs.serialization.json import ComplexSerializableType, serialize +from sec_certs.utils import helpers +from sec_certs.utils import parallel_processing as cert_processing +from sec_certs.utils.helpers import fips_dgst + +logger = logging.getLogger(__name__) + + +class FIPSAuxillaryDatasets(AuxillaryDatasets): + cpe_dset: CPEDataset | None = None + cve_dset: CVEDataset | None = None + algorithm_dset: FIPSAlgorithmDataset | None = None + + +class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxillaryDatasets], ComplexSerializableType): + """ + Class for processing of FIPSCertificate samples. Inherits from `ComplexSerializableType` and base abstract `Dataset` class. + """ + + def __init__( + self, + certs: dict[str, FIPSCertificate] = dict(), + root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH, + name: str | None = None, + description: str = "", + state: Dataset.DatasetInternalState | None = None, + auxillary_datasets: FIPSAuxillaryDatasets | None = None, + ): + self.certs = certs + self.timestamp = datetime.datetime.now() + self.sha256_digest = "not implemented" + self.name = name if name else type(self).__name__ + " dataset" + self.description = description if description else datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") + self.state = state if state else self.DatasetInternalState() + self.auxillary_datasets: FIPSAuxillaryDatasets = ( + auxillary_datasets if auxillary_datasets else FIPSAuxillaryDatasets() + ) + + self.root_dir = Path(root_dir) + + LIST_OF_CERTS_HTML: Final[dict[str, str]] = { + "fips_modules_active.html": constants.FIPS_ACTIVE_MODULES_URL, + "fips_modules_historical.html": constants.FIPS_HISTORICAL_MODULES_URL, + "fips_modules_revoked.html": constants.FIPS_REVOKED_MODULES_URL, + } + + @property + def policies_dir(self) -> Path: + return self.certs_dir / "policies" + + @property + def policies_pdf_dir(self) -> Path: + return self.policies_dir / "pdf" + + @property + def policies_txt_dir(self) -> Path: + return self.policies_dir / "txt" + + @property + def module_dir(self) -> Path: + return self.certs_dir / "modules" + + @property + def algorithm_dataset_path(self) -> Path: + return self.auxillary_datasets_dir / "algorithms.json" + + def __getitem__(self, item: str) -> FIPSCertificate: + try: + return super().__getitem__(item) + except KeyError: + return super().__getitem__(fips_dgst(item)) + + def _extract_data_from_html_modules(self) -> None: + """ + Extracts data from html module file + :param bool fresh: if all certs should be processed, or only the failed ones. Defaults to True + """ + logger.info("Extracting data from html modules.") + certs_to_process = [x for x in self if x.state.module_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + FIPSCertificate.parse_html_module, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting data from html modules", + ) + self.update_with_certs(processed_certs) + + @serialize + def extract_data(self) -> None: + logger.info("Extracting various data from certification artifacts.") + for cert in self: + cert.state.policy_extract_ok = True + cert.state.module_extract_ok = True + + self._extract_data_from_html_modules() + self._extract_policy_pdf_metadata() + self._extract_policy_pdf_keywords() + self._extract_algorithms_from_policy_tables() + + def _extract_policy_pdf_keywords(self) -> None: + logger.info("Extracting keywords from policy pdfs.") + certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + FIPSCertificate.extract_policy_pdf_keywords, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting keywords from policy pdfs", + ) + self.update_with_certs(processed_certs) + + def _download_all_artifacts_body(self, fresh: bool = True) -> None: + self._download_modules(fresh) + self._download_policies(fresh) + + def _download_modules(self, fresh: bool = True) -> None: + self.module_dir.mkdir(parents=True, exist_ok=True) + certs_to_process = [x for x in self if x.state.module_is_ok_to_download(fresh)] + + if fresh: + logger.info("Downloading HTML cryptographic modules.") + if not fresh and certs_to_process: + logger.info(f"Downloading {len(certs_to_process)} HTML modules for which download failed.") + + cert_processing.process_parallel( + FIPSCertificate.download_module, + certs_to_process, + config.n_threads, + progress_bar_desc="Downloading HTML modules", + ) + + def _download_policies(self, fresh: bool = True) -> None: + self.policies_pdf_dir.mkdir(parents=True, exist_ok=True) + certs_to_process = [x for x in self if x.state.policy_is_ok_to_download(fresh)] + + if fresh: + logger.info("Downloading PDF security policies.") + if not fresh and certs_to_process: + logger.info(f"Downloading {len(certs_to_process)} PDF security policies for which download failed.") + + cert_processing.process_parallel( + FIPSCertificate.download_policy, + certs_to_process, + config.n_threads, + progress_bar_desc="Downloading PDF security policies", + ) + + def _convert_all_pdfs_body(self, fresh: bool = True) -> None: + self._convert_policies_to_txt(fresh) + + def _convert_policies_to_txt(self, fresh: bool = True) -> None: + self.policies_txt_dir.mkdir(parents=True, exist_ok=True) + certs_to_process = [x for x in self if x.state.policy_is_ok_to_convert(fresh)] + + if fresh: + logger.info("Converting FIPS security policies to .txt") + if not fresh and certs_to_process: + logger.info( + f"Converting {len(certs_to_process)} FIPS security polcies to .txt for which previous convert failed." + ) + + cert_processing.process_parallel( + FIPSCertificate.convert_policy_pdf, + certs_to_process, + config.n_threads, + progress_bar_desc="Converting policies to pdf", + ) + + def _download_html_resources(self) -> None: + logger.info("Downloading HTML files that list FIPS certificates.") + html_urls = list(FIPSDataset.LIST_OF_CERTS_HTML.values()) + html_paths = [self.web_dir / x for x in FIPSDataset.LIST_OF_CERTS_HTML.keys()] + helpers.download_parallel(html_urls, html_paths) + + def _get_all_certs_from_html_sources(self) -> list[FIPSCertificate]: + return list( + itertools.chain.from_iterable( + self._get_certificates_from_html(self.web_dir / x) for x in self.LIST_OF_CERTS_HTML.keys() + ) + ) + + def _get_certificates_from_html(self, html_file: Path) -> list[FIPSCertificate]: + with open(html_file, encoding="utf-8") as handle: + html = BeautifulSoup(handle.read(), "html5lib") + + table = [x for x in html.find(id="searchResultsTable").tbody.contents if x != "\n"] + cert_ids: set[str] = set() + + for entry in table: + if isinstance(entry, NavigableString): + continue + cert_id = entry.find("a").text + if cert_id not in cert_ids: + cert_ids.add(cert_id) + + return [FIPSCertificate(cert_id) for cert_id in cert_ids] + + @classmethod + def from_web_latest(cls) -> FIPSDataset: + """ + Fetches the fresh snapshot of FIPSDataset from mirror. + """ + return cls.from_web(config.fips_latest_snapshot, "Downloading FIPS Dataset", "fips_latest_dataset.json") + + def _set_local_paths(self) -> None: + super()._set_local_paths() + if self.auxillary_datasets.algorithm_dset: + self.auxillary_datasets.algorithm_dset.json_path = self.algorithm_dataset_path + + cert: FIPSCertificate + for cert in self.certs.values(): + cert.set_local_paths(self.policies_pdf_dir, self.policies_txt_dir, self.module_dir) + + @serialize + def get_certs_from_web(self, to_download: bool = True, keep_metadata: bool = True) -> None: + self.web_dir.mkdir(parents=True, exist_ok=True) + + if to_download: + self._download_html_resources() + + logger.info("Adding unprocessed FIPS certificates into FIPSDataset.") + self.certs = {x.dgst: x for x in self._get_all_certs_from_html_sources()} + logger.info(f"The dataset now contains {len(self)} certificates.") + + if not keep_metadata: + shutil.rmtree(self.web_dir) + + self._set_local_paths() + self.state.meta_sources_parsed = True + + @serialize + def process_auxillary_datasets(self, download_fresh: bool = False) -> None: + super().process_auxillary_datasets(download_fresh) + self.auxillary_datasets.algorithm_dset = self._prepare_algorithm_dataset(download_fresh) + + def _prepare_algorithm_dataset(self, download_fresh_algs: bool = False) -> FIPSAlgorithmDataset: + logger.info("Preparing FIPSAlgorithm dataset.") + if not self.algorithm_dataset_path.exists() or download_fresh_algs: + alg_dset = FIPSAlgorithmDataset.from_web(self.algorithm_dataset_path) + alg_dset.to_json() + else: + alg_dset = FIPSAlgorithmDataset.from_json(self.algorithm_dataset_path) + + return alg_dset + + def _extract_algorithms_from_policy_tables(self): + logger.info("Extracting Algorithms from policy tables") + certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()] + cert_processing.process_parallel( + FIPSCertificate.get_algorithms_from_policy_tables, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting Algorithms from policy tables", + ) + + def _extract_policy_pdf_metadata(self) -> None: + logger.info("Extracting security policy metadata from the pdfs") + certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()] + processed_certs = cert_processing.process_parallel( + FIPSCertificate.extract_policy_pdf_metadata, + certs_to_process, + config.n_threads, + use_threading=False, + progress_bar_desc="Extracting security policy metadata", + ) + self.update_with_certs(processed_certs) + + def _compute_transitive_vulnerabilities(self) -> None: + logger.info("Computing heuristics: Computing transitive vulnerabilities in referenc(ed/ing) certificates.") + transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: cert.cert_id) + transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.policy_processed_references) + + for dgst in self.certs: + transitive_cve = transitive_cve_finder.predict_single_cert(dgst) + self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves + self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves + + def _prune_reference_candidates(self) -> None: + for cert in self: + cert.prune_referenced_cert_ids() + + # Previously, a following procedure was used to prune reference_candidates: + # - A set of algorithms was obtained via self.auxillary_datasets.algorithm_dset.get_algorithms_by_id(reference_candidate) + # - If any of these algorithms had the same vendor as the reference_candidate, the candidate was rejected + # - The rationale is that if an ID appears in a certificate s.t. an algorithm with the same ID was produced by the same vendor, the reference likely refers to alg. + # - Such reference should then be discarded. + # - We are uncertain of the effectivity of such measure, disabling it for now. + + def _compute_references(self, keep_unknowns: bool = False) -> None: + logger.info("Computing heuristics: Recovering references between certificates") + self._prune_reference_candidates() + + policy_reference_finder = ReferenceFinder() + policy_reference_finder.fit( + self.certs, lambda cert: cert.cert_id, lambda cert: cert.heuristics.policy_prunned_references + ) + + module_reference_finder = ReferenceFinder() + module_reference_finder.fit( + self.certs, lambda cert: cert.cert_id, lambda cert: cert.heuristics.module_prunned_references + ) + + for cert in self: + cert.heuristics.policy_processed_references = policy_reference_finder.predict_single_cert( + cert.dgst, keep_unknowns + ) + cert.heuristics.module_processed_references = module_reference_finder.predict_single_cert( + cert.dgst, keep_unknowns + ) + + def to_pandas(self) -> pd.DataFrame: + df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=FIPSCertificate.pandas_columns) + df = df.set_index("dgst") + + df.date_validation = pd.to_datetime(df.date_validation, infer_datetime_format=True) + df.date_sunset = pd.to_datetime(df.date_sunset, infer_datetime_format=True) + + # Manually delete one certificate with bad embodiment (seems to have many blank fields) + df = df.loc[~(df.embodiment == "*")] + + df = df.astype( + {"type": "category", "status": "category", "standard": "category", "embodiment": "category"} + ).fillna(value=np.nan) + + df.level = df.level.fillna(value=np.nan).astype("float") + # df.level = pd.Categorical(df.level, categories=sorted(df.level.dropna().unique().tolist()), ordered=True) + + # Introduce year when cert got valid + df["year_from"] = pd.DatetimeIndex(df.date_validation).year + + return df diff --git a/src/sec_certs/dataset/fips_algorithm.py b/src/sec_certs/dataset/fips_algorithm.py new file mode 100644 index 00000000..c48cff07 --- /dev/null +++ b/src/sec_certs/dataset/fips_algorithm.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import itertools +import logging +import re +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Iterator + +import pandas as pd +from bs4 import BeautifulSoup + +from sec_certs import constants +from sec_certs.dataset.json_path_dataset import JSONPathDataset +from sec_certs.sample import FIPSAlgorithm +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.utils import helpers + +logger = logging.getLogger(__name__) + + +class FIPSAlgorithmDataset(JSONPathDataset, ComplexSerializableType): + def __init__( + self, algs: dict[str, FIPSAlgorithm] = dict(), json_path: str | Path = constants.DUMMY_NONEXISTING_PATH + ): + self.algs = algs + self.json_path = Path(json_path) + self.alg_number_to_algs: dict[str, set[FIPSAlgorithm]] = dict() + + self._build_lookup_dicts() + + @property + def serialized_attributes(self) -> list[str]: + return ["algs"] + + def __iter__(self) -> Iterator[FIPSAlgorithm]: + yield from self.algs.values() + + def __getitem__(self, item: str) -> FIPSAlgorithm: + return self.algs.__getitem__(item) + + def __setitem__(self, key: str, value: FIPSAlgorithm) -> None: + self.algs.__setitem__(key, value) + + def __len__(self) -> int: + return len(self.algs) + + def __contains__(self, item: FIPSAlgorithm) -> bool: + if not isinstance(item, FIPSAlgorithm): + raise ValueError(f"{item} is not of FIPSAlgorithm class") + return item.dgst in self.algs.keys() and self.algs[item.dgst] == item + + def __eq__(self, other: object) -> bool: + return isinstance(other, FIPSAlgorithmDataset) and self.algs == other.algs + + @classmethod + def from_web(cls, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> FIPSAlgorithmDataset: + with TemporaryDirectory() as tmp_dir: + htmls = FIPSAlgorithmDataset.download_alg_list_htmls(Path(tmp_dir)) + algs = set(itertools.chain.from_iterable(FIPSAlgorithmDataset.parse_algorithms_from_html(x) for x in htmls)) + return cls({x.dgst: x for x in algs}, json_path) + + @staticmethod + def download_alg_list_htmls(output_dir: Path) -> list[Path]: + first_page_path = output_dir / "page1.html" + ITEMS_PER_PAGE = "ipp=250" + + res = helpers.download_file(constants.FIPS_ALG_SEARCH_URL + "1&" + ITEMS_PER_PAGE, first_page_path) + if res != constants.RESPONSE_OK: + res = helpers.download_file(constants.FIPS_ALG_SEARCH_URL + "1&" + ITEMS_PER_PAGE, first_page_path) + if res != constants.RESPONSE_OK: + logger.error(f"Could not build Algorithm dataset, got server response: {res}") + raise ValueError(f"Could not build Algorithm dataset, got server response: {res}") + + n_pages = FIPSAlgorithmDataset.get_number_of_html_pages(first_page_path) + + urls = [constants.FIPS_ALG_SEARCH_URL + str(i) + "&" + ITEMS_PER_PAGE for i in range(2, n_pages + 1)] + paths = [output_dir / f"page{i}.html" for i in range(2, n_pages + 1)] + responses = helpers.download_parallel(urls, paths, progress_bar_desc="Downloading FIPS Algorithm HTMLs") + + failed_tuples = [ + (url, path) for url, path, resp in zip(urls, paths, responses) if resp != constants.RESPONSE_OK + ] + if failed_tuples: + failed_urls, failed_paths = zip(*failed_tuples) + responses = helpers.download_parallel(failed_urls, failed_paths) + if any([x != constants.RESPONSE_OK for x in responses]): + raise ValueError("Failed to download the algorithms HTML data, the dataset won't be constructed.") + + return paths + + @staticmethod + def get_number_of_html_pages(html_path: Path) -> int: + with html_path.open("r") as handle: + soup = BeautifulSoup(handle, "html5lib") + return int(soup.select("span[data-total-pages]")[0].attrs["data-total-pages"]) + + @staticmethod + def parse_algorithms_from_html(html_path: Path) -> set[FIPSAlgorithm]: + df = pd.read_html(html_path)[0] + df["alg_type"] = df["Validation Number"].map(lambda x: re.sub(r"[0-9\s]", "", x)) + df["alg_number"] = df["Validation Number"].map(lambda x: re.sub(r"[^0-9]", "", x)) + df["alg"] = df.apply( + lambda row: FIPSAlgorithm( + row["alg_number"], row["alg_type"], row["Vendor"], row["Implementation"], row["Validation Date"] + ), + axis=1, + ) + return set(df["alg"]) + + def to_pandas(self) -> pd.DataFrame: + df = pd.DataFrame([x.pandas_tuple for x in self], columns=FIPSAlgorithm.pandas_columns) + df = df.set_index("dgst") + return df + + def _build_lookup_dicts(self) -> None: + for alg in self: + if alg.alg_number not in self.alg_number_to_algs: + self.alg_number_to_algs[alg.alg_number] = {alg} + else: + self.alg_number_to_algs[alg.alg_number].add(alg) + + def get_algorithms_by_id(self, alg_number: str) -> set[FIPSAlgorithm]: + return self.alg_number_to_algs.get(alg_number, set()) diff --git a/src/sec_certs/dataset/fips_iut.py b/src/sec_certs/dataset/fips_iut.py new file mode 100644 index 00000000..ce0f2f76 --- /dev/null +++ b/src/sec_certs/dataset/fips_iut.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Iterator, Mapping + +import requests + +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.dataset.dataset import logger +from sec_certs.dataset.json_path_dataset import JSONPathDataset +from sec_certs.sample.fips_iut import IUTSnapshot +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.utils.tqdm import tqdm + + +@dataclass +class IUTDataset(JSONPathDataset, ComplexSerializableType): + snapshots: list[IUTSnapshot] + _json_path: Path + + def __init__(self, snapshots: list[IUTSnapshot], json_path: str | Path = constants.DUMMY_NONEXISTING_PATH): + self.snapshots = snapshots + self.json_path = Path(json_path) + + def __iter__(self) -> Iterator[IUTSnapshot]: + yield from self.snapshots + + def __getitem__(self, item: int) -> IUTSnapshot: + return self.snapshots.__getitem__(item) + + def __len__(self) -> int: + return len(self.snapshots) + + @classmethod + def from_dumps(cls, dump_path: str | Path) -> IUTDataset: + directory = Path(dump_path) + fnames = list(directory.glob("*")) + snapshots = [] + for dump_path in tqdm(sorted(fnames), total=len(fnames)): + try: + snapshots.append(IUTSnapshot.from_dump(dump_path)) + except Exception as e: + logger.error(e) + return cls(snapshots) + + def to_dict(self) -> dict[str, list[IUTSnapshot]]: + return {"snapshots": list(self.snapshots)} + + @classmethod + def from_dict(cls, dct: Mapping) -> IUTDataset: + return cls(dct["snapshots"]) + + @classmethod + def from_web_latest(cls) -> IUTDataset: + """ + Get the IUTDataset from seccerts.org + """ + iut_resp = requests.get(config.fips_iut_dataset) + if iut_resp.status_code != 200: + raise ValueError(f"Getting IUT dataset failed: {iut_resp.status_code}") + with NamedTemporaryFile() as tmpfile: + tmpfile.write(iut_resp.content) + return cls.from_json(tmpfile.name) diff --git a/src/sec_certs/dataset/fips_mip.py b/src/sec_certs/dataset/fips_mip.py new file mode 100644 index 00000000..05ca5854 --- /dev/null +++ b/src/sec_certs/dataset/fips_mip.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Iterator, Mapping + +import requests + +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.dataset.dataset import logger +from sec_certs.dataset.json_path_dataset import JSONPathDataset +from sec_certs.sample.fips_mip import MIPSnapshot +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.utils.tqdm import tqdm + + +@dataclass +class MIPDataset(JSONPathDataset, ComplexSerializableType): + snapshots: list[MIPSnapshot] + _json_path: Path + + def __init__(self, snapshots: list[MIPSnapshot], json_path: str | Path = constants.DUMMY_NONEXISTING_PATH): + self.snapshots = snapshots + self.json_path = Path(json_path) + + def __iter__(self) -> Iterator[MIPSnapshot]: + yield from self.snapshots + + def __getitem__(self, item: int) -> MIPSnapshot: + return self.snapshots.__getitem__(item) + + def __len__(self) -> int: + return len(self.snapshots) + + @classmethod + def from_dumps(cls, dump_path: str | Path) -> MIPDataset: + directory = Path(dump_path) + fnames = list(directory.glob("*")) + snapshots = [] + for dump_path in tqdm(sorted(fnames), total=len(fnames)): + try: + snapshots.append(MIPSnapshot.from_dump(dump_path)) + except Exception as e: + logger.error(e) + return cls(snapshots) + + def to_dict(self) -> dict[str, list[MIPSnapshot]]: + return {"snapshots": list(self.snapshots)} + + @classmethod + def from_dict(cls, dct: Mapping) -> MIPDataset: + return cls(dct["snapshots"]) + + @classmethod + def from_web_latest(cls) -> MIPDataset: + """ + Get the MIPDataset from seccerts.org + """ + mip_resp = requests.get(config.fips_mip_dataset) + if mip_resp.status_code != 200: + raise ValueError(f"Getting MIP dataset failed: {mip_resp.status_code}") + with NamedTemporaryFile() as tmpfile: + tmpfile.write(mip_resp.content) + return cls.from_json(tmpfile.name) diff --git a/src/sec_certs/dataset/json_path_dataset.py b/src/sec_certs/dataset/json_path_dataset.py new file mode 100644 index 00000000..6eb4d968 --- /dev/null +++ b/src/sec_certs/dataset/json_path_dataset.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json +import logging +import shutil +from abc import ABC +from pathlib import Path + +from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, get_class_fullname + +logger = logging.getLogger(__name__) + + +class JSONPathDataset(ComplexSerializableType, ABC): + _json_path: Path + + @property + def json_path(self) -> Path: + return self._json_path + + @json_path.setter + def json_path(self, new_path: str | Path) -> None: + new_path = Path(new_path) + if new_path.is_dir(): + raise ValueError(f"Json path of {get_class_fullname(self)} cannot be a directory.") + + self._json_path = new_path + + def move_dataset(self, new_json_path: str | Path) -> None: + logger.info(f"Moving {get_class_fullname(self)} dataset to {new_json_path}") + new_json_path = Path(new_json_path) + new_json_path.parent.mkdir(parents=True, exist_ok=True) + + if self.json_path.exists(): + shutil.move(str(self.json_path), str(new_json_path)) + self.json_path = new_json_path + else: + self.json_path = new_json_path + self.to_json() + + @classmethod + def from_json(cls, input_path: str | Path): + with Path(input_path).open("r") as handle: + dset = json.load(handle, cls=CustomJSONDecoder) + dset.json_path = Path(input_path) + return dset diff --git a/src/sec_certs/dataset/protection_profile.py b/src/sec_certs/dataset/protection_profile.py new file mode 100644 index 00000000..edfb6850 --- /dev/null +++ b/src/sec_certs/dataset/protection_profile.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import json +import logging +import shutil +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import sec_certs.utils.helpers as helpers +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.sample.protection_profile import ProtectionProfile +from sec_certs.serialization.json import get_class_fullname + +logger = logging.getLogger(__name__) + + +@dataclass +class ProtectionProfileDataset: + pps: dict[tuple[str, str | None], ProtectionProfile] + _json_path: Path + + def __init__( + self, + pps: dict[tuple[str, str | None], ProtectionProfile], + json_path: str | Path = constants.DUMMY_NONEXISTING_PATH, + ) -> None: + self.pps = pps + self.json_path = Path(json_path) + + @property + def json_path(self): + return self._json_path + + @json_path.setter + def json_path(self, new_path: str | Path): + new_path = Path(new_path) + if new_path.is_dir(): + raise ValueError(f"Json path of {get_class_fullname(self)} cannot be a directory.") + + self._json_path = new_path + + def move_dataset(self, new_json_path: str | Path) -> None: + logger.info(f"Moving {get_class_fullname(self)} dataset to {new_json_path}") + new_json_path = Path(new_json_path) + new_json_path.parent.mkdir(parents=True, exist_ok=True) + + if not self.json_path.exists(): + raise ValueError("Cannot move the PPDataset if the json path does not exist.") + + shutil.move(str(self.json_path), str(new_json_path)) + self.json_path = new_json_path + + def __iter__(self): + yield from self.pps.values() + + def __getitem__(self, item: tuple[str, str | None]) -> ProtectionProfile: + return self.pps.__getitem__(item) + + def __setitem__(self, key: tuple[str, str | None], value: ProtectionProfile): + self.pps.__setitem__(key, value) + + def __contains__(self, key): + return key in self.pps + + def __len__(self) -> int: + return len(self.pps) + + @classmethod + def from_json(cls, json_path: str | Path): + with Path(json_path).open("r") as handle: + data = json.load(handle) + pps = [ProtectionProfile.from_old_api_dict(x) for x in data.values()] + + dct = {} + for item in pps: + if (item.pp_name, item.pp_link) in dct: + logger.warning(f"Duplicate entry in PP dataset: {(item.pp_name, item.pp_link)}") + dct[(item.pp_name, item.pp_link)] = item + + return cls(dct) + + @classmethod + def from_web(cls, store_dataset_path: Path | None = None): + + logger.info(f"Downloading static PP dataset from: {config.pp_latest_snapshot}") + if not store_dataset_path: + tmp = tempfile.TemporaryDirectory() + store_dataset_path = Path(tmp.name) / "pp_dataset.json" + + helpers.download_file(config.pp_latest_snapshot, store_dataset_path) + obj = cls.from_json(store_dataset_path) + + if not store_dataset_path: + tmp.cleanup() + + return obj diff --git a/src/sec_certs/model/__init__.py b/src/sec_certs/model/__init__.py new file mode 100644 index 00000000..fe1024f9 --- /dev/null +++ b/src/sec_certs/model/__init__.py @@ -0,0 +1,11 @@ +""" +This package exposes model (mostly transformers and classifiers) that apply complex transformations. These are to be +leveraged by members of Dataset package and are directly applied on members of Sample class (or on built-in objects). +""" + +from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.model.reference_finder import ReferenceFinder +from sec_certs.model.sar_transformer import SARTransformer +from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder + +__all__ = ["CPEClassifier", "ReferenceFinder", "TransitiveVulnerabilityFinder", "SARTransformer"] diff --git a/src/sec_certs/model/cpe_matching.py b/src/sec_certs/model/cpe_matching.py new file mode 100644 index 00000000..0febea5d --- /dev/null +++ b/src/sec_certs/model/cpe_matching.py @@ -0,0 +1,404 @@ +from __future__ import annotations + +import itertools +import logging +import operator +import re +from typing import Pattern + +import spacy +from rapidfuzz import fuzz +from sklearn.base import BaseEstimator + +from sec_certs import cert_rules, constants +from sec_certs.sample.cpe import CPE +from sec_certs.utils.tqdm import tqdm + +logger = logging.getLogger(__name__) + + +class CPEClassifier(BaseEstimator): + """ + Class that can predict CPE matches for certificate instances. + Adheres to sklearn BaseEstimator interface. + Fit method is called on list of CPEs and build two look-up dictionaries, see description of attributes. + """ + + vendor_to_versions_: dict[str, set[str]] # Key: CPE vendor, Value: versions of all CPE records of that vendor + vendor_version_to_cpe_: dict[tuple[str, str], set[CPE]] # Key: (CPE vendor, version), Value: CPEs (vendor, version) + vendors_: set[str] + + def __init__(self, match_threshold: int = 80, n_max_matches: int = 10, spacy_model_to_use: str = "en_core_web_sm"): + self.match_threshold = match_threshold + self.n_max_matches = n_max_matches + self.nlp = spacy.load(spacy_model_to_use, disable=["parser", "ner"]) + + def fit(self, X: list[CPE], y: list[str] | None = None) -> CPEClassifier: + """ + Just creates look-up structures from provided list of CPEs + + :param List[CPE] X: List of CPEs that can be matched with predict() + :param Optional[List[str]] y: will be ignored, specified to adhere to sklearn BaseEstimator interface, defaults to None + :return CPEClassifier: return self to allow method chaining + """ + self._build_lookup_structures(X) + return self + + @staticmethod + def _filter_short_cpes(cpes: list[CPE]) -> list[CPE]: + """ + Short CPE items are super easy to match with 100% rank, but they are hardly informative. This method discards them. + + :param List[CPE] cpes: List of CPEs to filtered + :return List[CPE]: All CPEs in cpes variable which item name has at least 4 characters. + """ + return list(filter(lambda x: x.item_name is not None and len(x.item_name) > 3, cpes)) + + def _build_lookup_structures(self, X: list[CPE]) -> None: + """ + Builds several look-up dictionaries for fast matching. + - vendor_to_version_: each vendor is mapped to set of versions that appear in combination with vendor in CPE dataset + - vendor_version_to_cpe_: Each (vendor, version) tuple is mapped to a set of CPE items that appear in combination with this tuple in CPE dataset + - vendors_: Just aggregates set of vendors, used for prunning later on. + + :param List[CPE] X: List of CPEs that will be used to build the dictionaries + """ + sufficiently_long_cpes = self._filter_short_cpes(X) + self.vendor_to_versions_ = {x.vendor: set() for x in sufficiently_long_cpes} + self.vendors_ = set(self.vendor_to_versions_.keys()) + self.vendor_version_to_cpe_ = dict() + + for cpe in tqdm(sufficiently_long_cpes, desc="Fitting the CPE classifier"): + self.vendor_to_versions_[cpe.vendor].add(cpe.version) + if (cpe.vendor, cpe.version) not in self.vendor_version_to_cpe_: + self.vendor_version_to_cpe_[(cpe.vendor, cpe.version)] = {cpe} + else: + self.vendor_version_to_cpe_[(cpe.vendor, cpe.version)].add(cpe) + + def predict(self, X: list[tuple[str, str, str]]) -> list[set[str] | None]: + """ + Will predict CPE uris for List of Tuples (vendor, product name, identified versions in product name) + + :param List[Tuple[str, str, str]] X: tuples (vendor, product name, identified versions in product name) + :return List[Optional[Set[str]]]: List of CPE uris that correspond to given input, None if nothing was found. + """ + return [self.predict_single_cert(x[0], x[1], x[2]) for x in tqdm(X, desc="Predicting")] + + def predict_single_cert( + self, + vendor: str | None, + product_name: str, + versions: set[str], + relax_version: bool = False, + relax_title: bool = False, + ) -> set[str] | None: + """ + Predict List of CPE uris for triplet (vendor, product_name, list_of_versions). The prediction is made as follows: + 1. Sanitize vendor name, lemmatize product name. + 2. Find vendors in CPE dataset that are related to the certificate + 3. Based on (vendors, versions) find all CPE items that are considered as candidates for match + 4. Compute string similarity of the candidate CPE matches and certificate name + 5. Evaluate best string similarity, if above threshold, declare it a match. + 6. If no CPE item is matched, try again but relax version and check CPEs that don't have their version specified. + 7. (Also, search for 100% CPE matches on item name instead of title.) + + :param Optional[str] vendor: manufacturer of the certificate + :param str product_name: name of the certificate + :param Set[str] versions: List of versions that appear in the certificate name + :param bool relax_version: See step 6 above., defaults to False + :param bool relax_title: See step 7 above, defaults to False + :return Optional[Set[str]]: Set of matching CPE uris, None if no matches found + """ + lemmatized_product_name = self._lemmatize_product_name(product_name) + candidate_vendors = self._get_candidate_list_of_vendors( + CPEClassifier._discard_trademark_symbols(vendor).lower() if vendor else vendor + ) + candidates = self._get_candidate_cpe_matches(candidate_vendors, versions) + candidates = self._filter_candidates_by_platform(candidates, product_name) + candidates = self._filter_candidates_by_update(candidates, lemmatized_product_name) + + ratings = [ + self._compute_best_match(cpe, lemmatized_product_name, candidate_vendors, versions, relax_title=relax_title) + for cpe in candidates + ] + threshold = self.match_threshold if not relax_version else 100 + final_matches_aux: list[tuple[float, CPE]] = list(filter(lambda x: x[0] >= threshold, zip(ratings, candidates))) + final_matches_aux = sorted(final_matches_aux, key=operator.itemgetter(0, 1), reverse=True) + final_matches: set[str] | None = { + x[1].uri for x in final_matches_aux[: self.n_max_matches] if x[1].uri is not None + } + + if not relax_title and not final_matches: + final_matches = self.predict_single_cert( + vendor, product_name, versions, relax_version=relax_version, relax_title=True + ) + + if not relax_version and not final_matches: + final_matches = self.predict_single_cert( + vendor, product_name, {constants.CPE_VERSION_NA}, relax_version=True, relax_title=relax_title + ) + + return final_matches if final_matches else None + + def _filter_candidates_by_update(self, cpes: list[CPE], cert_title: str) -> list[CPE]: + """ + Update means `service pack` or `release`. + """ + + def filter_condition(regex: Pattern, cpe: CPE, min_value: int, soft: bool = True): + if matches := re.findall(regex, cpe.update): + return int(re.findall(r"\d+", matches[0])[0]) >= min_value + return True if soft else False + + update_regexes = [cert_rules.SERVICE_PACK_RE, cert_rules.RELEASE_RE] + + for update_regex in update_regexes: + if matches := re.findall(update_regex, cert_title): + min_value = min([int(re.findall(r"\d+", x)[0]) for x in matches]) + soft = not any(re.search(update_regex, cpe.update + str(cpe.title)) for cpe in cpes) + return [x for x in cpes if filter_condition(update_regex, x, min_value, soft)] + + return cpes + + def _filter_candidates_by_platform(self, cpes: list[CPE], cert_title: str) -> list[CPE]: + def filter_condition(cpe: CPE, cert_platforms: set[str]): + if not cert_platforms and cpe.target_hw == "*": + return True + if cert_platforms and cpe.target_hw == "*": + return any(re.search(cert_rules.PLATFORM_REGEXES[x], str(cpe.title)) for x in cert_platforms) + if not cert_platforms and cpe.target_hw != "*": + return False + if cert_platforms and cpe.target_hw != "*": + target_hw_platforms = [ + platform + for platform, regex in cert_rules.PLATFORM_REGEXES.items() + if re.search(regex, cpe.target_hw) + ] + assert len(target_hw_platforms) <= 1 + can_return_true = any( + re.search(cert_rules.PLATFORM_REGEXES[x], cpe.target_hw + str(cpe.title)) for x in cert_platforms + ) + if not target_hw_platforms: + return can_return_true + else: + return can_return_true and target_hw_platforms[0] in cert_platforms + + crt_platforms = { + platform for platform, regex in cert_rules.PLATFORM_REGEXES.items() if re.search(regex, cert_title) + } + return [x for x in cpes if filter_condition(x, crt_platforms)] + + def _compute_best_match( + self, + cpe: CPE, + product_name: str, + candidate_vendors: set[str] | None, + versions: set[str], + relax_title: bool = False, + ) -> float: + """ + Tries several different settings in which string similarity between CPE and certificate name is tested. + For definition of string similarity, see rapidfuzz package on GitHub. Both token set ratio and partial ratio are tested, + always both on CPE title and CPE item name. + + :param CPE cpe: CPE to test + :param str product_name: name of the certificate + :param Optional[Set[str]] candidate_vendors: vendors that appear in the certificate + :param Set[str] versions: versions that appear in the certificate + :param bool relax_title: if to relax title or not, defaults to False + :return float: Maximal value of the four string similarities discussed above. + """ + if relax_title: + sanitized_title = ( + CPEClassifier._fully_sanitize_string(cpe.title) + if cpe.title + else CPEClassifier._fully_sanitize_string( + cpe.vendor + " " + cpe.item_name + " " + cpe.version + " " + cpe.update + " " + cpe.target_hw + ) + ) + else: + if cpe.title: + sanitized_title = CPEClassifier._fully_sanitize_string(cpe.title) + else: + return 0 + + # Sometimes, sanitization shortens CPE title to very short length. E.g., CPEs in Japanese unicode symbols that get all deteled. + if len(sanitized_title) < 5: + return 0 + + sanitized_item_name = CPEClassifier._fully_sanitize_string(cpe.item_name) + sanitized_cpe_stripped_manufacturer = re.sub(r"\b" + rf"{cpe.vendor}" + r"\b", "", sanitized_title) + standard_version_product_name = self._standardize_version_in_cert_name(product_name, versions) + + # The expression below is currently unused, it could assist with some matches though + # cert_stripped = CPEClassifier._strip_manufacturer_and_version(product_name, candidate_vendors, versions) + + # On some ratings, we require 100 match regardless of the treshold in settings. + ratings = [ + fuzz.token_set_ratio(product_name, sanitized_title), + fuzz.token_set_ratio(standard_version_product_name, sanitized_title), + fuzz.partial_token_sort_ratio(product_name, sanitized_title, score_cutoff=100), + fuzz.partial_token_sort_ratio(standard_version_product_name, sanitized_title, score_cutoff=100), + fuzz.partial_ratio(product_name, sanitized_title, score_cutoff=100), + fuzz.partial_ratio(standard_version_product_name, sanitized_title, score_cutoff=100), + ] + + # Big-IP has dumb CPEs that contain only that string in item name, which leads to false positives. + if relax_title and cpe.item_name != "big-ip": + ratings += [ + fuzz.token_set_ratio(product_name, sanitized_cpe_stripped_manufacturer, score_cutoff=100), + fuzz.partial_ratio(product_name, sanitized_cpe_stripped_manufacturer, score_cutoff=100), + fuzz.token_set_ratio(product_name, sanitized_item_name, score_cutoff=100), + fuzz.partial_ratio(product_name, sanitized_item_name, score_cutoff=100), + ] + + return max(ratings) + + @staticmethod + def _fully_sanitize_string(string: str) -> str: + return CPEClassifier._replace_special_chars_with_space( + CPEClassifier._discard_trademark_symbols(string.lower()) + ).strip() + + @staticmethod + def _replace_special_chars_with_space(string: str) -> str: + return re.sub(r"[^a-zA-Z0-9 \n\.]", " ", string) + + @staticmethod + def _discard_trademark_symbols(string: str) -> str: + return string.replace("®", "").replace("™", "") + + @staticmethod + def _strip_manufacturer_and_version(string: str, manufacturers: set[str] | None, versions: set[str]) -> str: + to_strip = versions | manufacturers if manufacturers else versions + for x in to_strip: + string = string.lower().replace(CPEClassifier._replace_special_chars_with_space(x.lower()), " ").strip() + return string + + @staticmethod + def _standardize_version_in_cert_name(string: str, detected_versions: set[str]) -> str: + for ver in detected_versions: + version_regex = r"(" + r"(\bversion)\s*" + ver + r"+) | (\bv\s*" + ver + r"+)" + string = re.sub(version_regex, " " + ver + " ", string, flags=re.IGNORECASE) + return string + + def _process_manufacturer(self, manufacturer: str, result: set) -> set[str]: + tokenized = manufacturer.split() + if tokenized[0] in self.vendors_: + result.add(tokenized[0]) + if len(tokenized) > 1 and tokenized[0] + tokenized[1] in self.vendors_: + result.add(tokenized[0] + tokenized[1]) + + # Below are completely manual fixes + if "hewlett" in tokenized or "hewlett-packard" in tokenized or manufacturer == "hewlett packard": + result.add("hp") + if "thales" in tokenized: + result.add("thalesesecurity") + result.add("thalesgroup") + if "stmicroelectronics" in tokenized: + result.add("st") + if "athena" in tokenized and "smartcard" in tokenized: + result.add("athena-scs") + if tokenized[0] == "the" and not result: + candidate_result = self._get_candidate_list_of_vendors(" ".join(tokenized[1:])) + return set(candidate_result) if candidate_result else set() + + return set(result) if result else set() + + def _get_candidate_list_of_vendors(self, manufacturer: str | None) -> set[str]: + """ + Given manufacturer name, this method will find list of plausible vendors from CPE dataset that are likely related. + + :param Optional[str] manufacturer: manufacturer + :return Set[str]: List of related manufacturers, None if nothing relevant is found. + """ + result: set[str] = set() + if not manufacturer: + return result + + splits = re.compile(r"[,/]").findall(manufacturer) + + if splits: + vendor_tokens = set( + itertools.chain.from_iterable([x.strip() for x in manufacturer.split(s)] for s in splits) + ) + result_aux = [self._get_candidate_list_of_vendors(x) for x in vendor_tokens] + result_used = set(set(itertools.chain.from_iterable(x for x in result_aux if x))) + return result_used if result_used else set() + + if manufacturer in self.vendors_: + result.add(manufacturer) + + return self._process_manufacturer(manufacturer, result) + + def _get_candidate_vendor_version_pairs( + self, cert_candidate_cpe_vendors: set[str], cert_candidate_versions: set[str] + ) -> list[tuple[str, str]] | None: + """ + Given parameters, will return Pairs (cpe_vendor, cpe_version) that are relevant to a given sample + + + :param Set[str] cert_candidate_cpe_vendors: list of CPE vendors relevant to a sample + :param Set[str] cert_candidate_versions: List of versions heuristically extracted from the sample name + :return Optional[List[Tuple[str, str]]]: List of tuples (cpe_vendor, cpe_version) that can be used in the lookup table to search the CPE dataset. + """ + + def is_cpe_version_among_cert_versions(cpe_version: str | None, cert_versions: set[str]) -> bool: + def simple_startswith(seeked_version: str, checked_string: str) -> bool: + if seeked_version == checked_string: + return True + else: + return ( + checked_string.startswith(seeked_version) and not checked_string[len(seeked_version)].isdigit() + ) + + if not cpe_version: + return False + just_numbers = r"(\d{1,5})(\.\d{1,5})" + + # This assures that on cert version with at least two tokens, we don't match only one-token CPE. + # E.g. cert with version 7.6 must not match CPE record of version 7 + if len(cert_versions) == 1 and len(list(cert_versions)[0]) >= 3 and len(cpe_version) < 3: + return False + + # Except from startswith stuff, this also mandates that for long enough cert vesions (e.g. `3.1`) we do not + # match too short CPE versions, e.g. `3` + for v in cert_versions: + if ( + (simple_startswith(v, cpe_version) and re.search(just_numbers, cpe_version)) + or simple_startswith(cpe_version, v) + ) and (len(v) < 3 or len(cpe_version) >= 3): + return True + return False + + if not cert_candidate_cpe_vendors: + return None + + candidate_vendor_version_pairs: list[tuple[str, str]] = [] + for vendor in cert_candidate_cpe_vendors: + viable_cpe_versions = self.vendor_to_versions_.get(vendor, set()) + matched_cpe_versions = [ + x for x in viable_cpe_versions if is_cpe_version_among_cert_versions(x, cert_candidate_versions) + ] + candidate_vendor_version_pairs.extend([(vendor, x) for x in matched_cpe_versions]) + return candidate_vendor_version_pairs + + def _get_candidate_cpe_matches(self, candidate_vendors: set[str], candidate_versions: set[str]) -> list[CPE]: + """ + Given List of candidate vendors and candidate versions found in certificate, candidate CPE matches are found + + :param Set[str] candidate_vendors: List of version strings that were found in the certificate + :param Set[str] candidate_versions: List of vendor strings that were found in the certificate + :return List[CPE]: List of CPE records that could match, to be refined later + """ + candidate_vendor_version_pairs = self._get_candidate_vendor_version_pairs(candidate_vendors, candidate_versions) + return ( + list(itertools.chain.from_iterable(self.vendor_version_to_cpe_[x] for x in candidate_vendor_version_pairs)) + if candidate_vendor_version_pairs + else [] + ) + + def _lemmatize_product_name(self, product_name: str) -> str: + if not product_name: + return product_name + return " ".join([token.lemma_ for token in self.nlp(CPEClassifier._fully_sanitize_string(product_name))]) diff --git a/src/sec_certs/model/evaluation.py b/src/sec_certs/model/evaluation.py new file mode 100644 index 00000000..dd5eaffd --- /dev/null +++ b/src/sec_certs/model/evaluation.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import numpy as np + +import sec_certs.utils.helpers as helpers +from sec_certs.dataset.cpe import CPEDataset +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.fips import FIPSCertificate +from sec_certs.serialization.json import CustomJSONEncoder + +logger = logging.getLogger(__name__) + + +def get_validation_dgsts(filepath: str | Path) -> set[str]: + with Path(filepath).open("r") as handle: + return set(json.load(handle)) + + +def compute_precision(y: np.ndarray, y_pred: np.ndarray, **kwargs) -> float: + prec = [] + for true, pred in zip(y, y_pred): + set_pred = set(pred) if pred else set() + set_true = set(true) if true else set() + if not set_pred and not set_true: + pass + else: + prec.append(len(set_true) / len(set_true.union(set_pred))) + return np.mean(prec) # type: ignore + + +def evaluate( + x_valid: list[CommonCriteriaCert | FIPSCertificate], + y_valid: list[set[str] | None], + outpath: Path | str | None, + cpe_dset: CPEDataset, +) -> None: + y_pred = [x.heuristics.cpe_matches for x in x_valid] + precision = compute_precision(np.array(y_valid), np.array(y_pred)) + + correctly_classified = [] + badly_classified = [] + n_cpes_lost = 0 + n_certs_has_lost_cpes = 0 + n_certs_lost = 0 + + for cert, predicted_cpes, verified_cpes in zip(x_valid, y_pred, y_valid): + if not verified_cpes: + verified_cpes = set() + verified_cpes_dict = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in verified_cpes} + + if not predicted_cpes: + predicted_cpes = set() + predicted_cpes_dict = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in predicted_cpes} + + cert_name = cert.name if isinstance(cert, CommonCriteriaCert) else cert.web_data.module_name + vendor = cert.manufacturer if isinstance(cert, CommonCriteriaCert) else cert.web_data.vendor + + should_be_removed = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in predicted_cpes - verified_cpes} + should_be_added = {x: cpe_dset[x].title if cpe_dset[x].title else x for x in verified_cpes - predicted_cpes} + + record = { + "certificate_name": cert_name, + "vendor": vendor, + "heuristic version": helpers.compute_heuristics_version(cert_name) if cert_name else None, + "predicted_cpes": predicted_cpes_dict, + "manually_assigned_cpes": verified_cpes_dict, + "should_be_removed": should_be_removed, + "should_be_added": should_be_added, + } + + if not predicted_cpes and verified_cpes: + n_certs_lost += 1 + + if predicted_cpes.issubset(verified_cpes): + correctly_classified.append(record) + else: + badly_classified.append(record) + + if should_be_added: + n_certs_has_lost_cpes += 1 + n_cpes_lost += len(should_be_added) + + results = { + "Precision": precision, + "n_cpes_lost": n_cpes_lost, + "n_certs_has_lost_cpes": n_certs_has_lost_cpes, + "n_certs_lost": n_certs_lost, + "correctly_classified": correctly_classified, + "badly_classified": badly_classified, + } + logger.info( + f"Precision: {precision}; the classifier now misses {n_cpes_lost} CPE matches from {n_certs_has_lost_cpes} certificates ({n_certs_lost} certificates no longer have a match). In total, {sum([len(x) for x in y_pred if x])} CPEs were matched in {len(y_pred)} certs." + ) + + if outpath: + with Path(outpath).open("w") as handle: + json.dump(results, handle, indent=4, cls=CustomJSONEncoder, sort_keys=True) diff --git a/src/sec_certs/model/reference_finder.py b/src/sec_certs/model/reference_finder.py new file mode 100644 index 00000000..117a8b9d --- /dev/null +++ b/src/sec_certs/model/reference_finder.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from typing import Callable, Dict, List, Optional, Set, TypeVar + +from sec_certs.sample.certificate import Certificate, References + +CertSubType = TypeVar("CertSubType", bound=Certificate) +Certificates = Dict[str, CertSubType] +ReferencedByDirect = Dict[str, Set[str]] +ReferencedByIndirect = Dict[str, Set[str]] +ReferencesType = Dict[str, Dict[str, Optional[Set[str]]]] +IDMapping = Dict[str, List[str]] +UnknownReferences = Dict[str, Set[str]] +IDLookupFunc = Callable[[CertSubType], str] +ReferenceLookupFunc = Callable[[CertSubType], Set[str]] + + +# TODO: All of this can and should be rewritten on top of networkx or some other graph library. +class ReferenceFinder: + """ + The class assigns references of other certificate instances for each instance. + Adheres to sklearn BaseEstimator interface. + The fit is called on a dictionary of certificates, builds a hashmap of references, and assigns references for each certificate in the dictionary. + """ + + def __init__(self): + self.references: ReferencesType = {} + self.id_mapping: IDMapping = {} + self._fitted: bool = False + + def _create_id_mapping(self, certificates: Certificates, id_func: IDLookupFunc) -> None: + """ + Create the ID mapping of certificate IDs to certificate digests. + + Necessary for handling duplicates. + """ + # Create a mapping of certificate ID to certificate digests with that ID. + for dgst in certificates: + cert_id = id_func(certificates[dgst]) + c_list = self.id_mapping.setdefault(cert_id, []) + c_list.append(dgst) + + # Sort digests in ID mapping to have deterministic behavior. + # The certificate with the first digest will be used with that ID, others will be discarded. + for digests in self.id_mapping.values(): + digests.sort() + + def _compute_indirect_references(self, referenced_by: ReferencedByDirect) -> ReferencedByIndirect: + """ + Compute indirect references via a BFS algorithm. + """ + referenced_by_indirect: ReferencedByIndirect = {} + + # Populate with direct references. + certs_id_list = referenced_by.keys() + for cert_id in certs_id_list: + referenced_by_indirect[cert_id] = set() + for item in referenced_by[cert_id]: + referenced_by_indirect[cert_id].add(item) + + # Flood in the indirect ones. + new_change_detected = True + while new_change_detected: + new_change_detected = False + + for cert_id in certs_id_list: + tmp_referenced_by_indirect_nums = referenced_by_indirect[cert_id].copy() + for referencing in tmp_referenced_by_indirect_nums: + if referencing in certs_id_list: + tmp_referencing = referenced_by_indirect[referencing].copy() + newly_discovered_references = [ + x for x in tmp_referencing if x not in referenced_by_indirect[cert_id] + ] + referenced_by_indirect[cert_id].update(newly_discovered_references) + if newly_discovered_references: + new_change_detected = True + return referenced_by_indirect + + def _build_referenced_by( + self, certificates: Certificates, ref_lookup_func: ReferenceLookupFunc + ) -> tuple[ReferencedByDirect, ReferencedByIndirect]: + referenced_by: ReferencedByDirect = {} + + for this_cert_id, cert_digests in self.id_mapping.items(): + # Take the first certificate digest from the ID mapping (to ensure deterministic behavior and resolve duplicates). + # TODO: A better approach for handling duplicates in the future would be nice. + cert_dgst = cert_digests[0] + cert_obj = certificates[cert_dgst] + + refs = ref_lookup_func(cert_obj) + if refs is None: + continue + + # Process direct reference + # All are added here, the unknown ones are filtered later on. + for cert_id in refs: + if cert_id == this_cert_id: + continue + referenced_by.setdefault(cert_id, set()) + referenced_by[cert_id].add(this_cert_id) + + # Now do the indirect ones + referenced_by_indirect = self._compute_indirect_references(referenced_by) + return referenced_by, referenced_by_indirect + + def _get_reverse_references( + self, cert_id: str, references: ReferencedByDirect | ReferencedByIndirect + ) -> set[str] | None: + result = set() + + for other_id in references: + if cert_id in references[other_id]: + result.add(other_id) + + return result if result else None + + def _build_referencing( + self, referenced_by_direct: ReferencedByDirect, referenced_by_indirect: ReferencedByIndirect + ) -> None: + for cert_id, cert_digests in self.id_mapping.items(): + cert_dgst = cert_digests[0] + self.references[cert_dgst] = { + "directly_referenced_by": referenced_by_direct.get(cert_id, None), + "indirectly_referenced_by": referenced_by_indirect.get(cert_id, None), + "directly_referencing": self._get_reverse_references(cert_id, referenced_by_direct), + "indirectly_referencing": self._get_reverse_references(cert_id, referenced_by_indirect), + } + + def fit(self, certificates: Certificates, id_func: IDLookupFunc, ref_lookup_func: ReferenceLookupFunc) -> None: + """ + Builds a list of references and assigns references for each certificate instance. + + :param Certificates certificates: dictionary of certificates with hashes as key + :param IDLookupFunc id_func: lookup function for cert id + :param ReferenceLookupFunc ref_lookup_func: lookup for references + """ + if self._fitted: + raise ValueError("Finder already fitted") + # Create the ID mapping first so that we can resolve duplicates. + self._create_id_mapping(certificates, id_func) + + # Build the referenced_by first + referenced_by_direct, referenced_by_indirect = self._build_referenced_by(certificates, ref_lookup_func) + + # Build the referencing second (this actually writes into self.references). + self._build_referencing(referenced_by_direct, referenced_by_indirect) + self._fitted = True + + @property + def unknown_references(self) -> UnknownReferences: + """ + Get the unknown references in the fitted dataset (to unknown certificate IDs, not in the dataset during fit). + """ + if not self._fitted: + return {} + result = {} + for cert_id, digests in self.id_mapping.items(): + cert_digest = digests[0] + cert_references = self.references[cert_digest] + direct_refs = cert_references["directly_referencing"] + if not direct_refs: + continue + unknowns = set(filter(lambda refd_cert_id: refd_cert_id not in self.id_mapping, direct_refs)) + if unknowns: + result[cert_id] = unknowns + return result + + @property + def duplicates(self) -> IDMapping: + """ + Get the duplicates in the fitted dataset. + + :return IDMapping: Mapping of certificate ID to digests that share it. + """ + if not self._fitted: + return {} + return {cert_id: digests for cert_id, digests in self.id_mapping.items() if len(digests) > 1} + + def predict_single_cert(self, dgst: str, keep_unknowns: bool = True) -> References: + """ + Get the references object for specified certificate digest. + + :param dgst: certificate digest + :param keep_unknowns: Whether to keep references to unknown certificate IDs + :return References: References object + """ + if not self._fitted: + raise ValueError("Finder not yet fitted") + + def wrap(res): + if not res: + return None + # If we do not want the unknown references, filter them here. + if not keep_unknowns: + res = set(filter(lambda cert_id: cert_id in self.id_mapping, res)) + return set(res) if res else None + + if dgst not in self.references: + return References() + + return References( + wrap(self.references[dgst].get("directly_referenced_by", None)), + wrap(self.references[dgst].get("indirectly_referenced_by", None)), + wrap(self.references[dgst].get("directly_referencing", None)), + wrap(self.references[dgst].get("indirectly_referencing", None)), + ) + + def predict(self, dgst_list: list[str], keep_unknowns: bool = True) -> dict[str, References]: + """ + Get the references for a list of certificate digests. + + :param dgst_list: List of certificate digests. + :param keep_unknowns: Whether to keep references to and from unknown certificate IDs + :return Dict[str, References]: Dict with certificate hash and References object. + """ + if not self._fitted: + raise ValueError("Finder not yet fitted") + cert_references = {} + + for dgst in dgst_list: + cert_references[dgst] = self.predict_single_cert(dgst, keep_unknowns=keep_unknowns) + + return cert_references diff --git a/src/sec_certs/model/sar_transformer.py b/src/sec_certs/model/sar_transformer.py new file mode 100644 index 00000000..a5bcff57 --- /dev/null +++ b/src/sec_certs/model/sar_transformer.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +from typing import Dict, Iterable, cast + +from sklearn.base import BaseEstimator, TransformerMixin + +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.sar import SAR, SAR_DICT_KEY + +logger = logging.getLogger(__name__) + + +# TODO: Right now we ignore number of ocurrences for final SAR selection. If we keep it this way, we can discard that variable +class SARTransformer(BaseEstimator, TransformerMixin): + """ + Class for transforming SARs defined in st_keywords and report_keywords dictionaries into SAR objects. + This class implements sklearn transformer interface, so fit_transform() can be called on it. + """ + + def fit(self, certificates: Iterable[CommonCriteriaCert]) -> SARTransformer: + """ + Just returns self, no fitting needed + + :param Iterable[CommonCriteriaCert] certificates: Unused parameter + :return SARTransformer: return self + """ + return self + + def transform(self, certificates: Iterable[CommonCriteriaCert]) -> list[set[SAR] | None]: + """ + Just a wrapper around transform_single_cert() called on an iterable of CommonCriteriaCert. + + :param Iterable[CommonCriteriaCert] certificates: Iterable of CommonCriteriaCert objects to perform the extraction on. + :return List[Optional[Set[SAR]]]: Returns List of results from transform_single_cert(). + """ + return [self.transform_single_cert(cert) for cert in certificates] + + def transform_single_cert(self, cert: CommonCriteriaCert) -> set[SAR] | None: + """ + Given CommonCriteriaCert, will transform SAR keywords extracted from txt files + into a set of SAR objects. Also handles extractin of correct SAR levels, duplicities and filtering. + Uses three sources: CSV scan, security target, and certification report. + The caller should assure that the certificates have the keywords extracted. + + :param CommonCriteriaCert cert: Certificate to extract SARs from + :return Optional[Set[SAR]]: Set of SARs, None if none were identified. + """ + sec_level_candidates, st_candidates, report_candidates = self._collect_sar_candidates_from_all_sources(cert) + return self._resolve_candidate_conflicts(sec_level_candidates, st_candidates, report_candidates, cert.dgst) + + @staticmethod + def _collect_sar_candidates_from_all_sources(cert: CommonCriteriaCert) -> tuple[set[SAR], set[SAR], set[SAR]]: + """ + Parses SARs from three distinct sources and returns the results as a three tuple: + - Security level from CSV scan + - Keywords from Security target + - Keywords from Certification report + """ + + def st_keywords_may_have_sars(sample: CommonCriteriaCert): + return sample.pdf_data.st_keywords and SAR_DICT_KEY in sample.pdf_data.st_keywords + + def report_keywords_may_have_sars(sample: CommonCriteriaCert): + return sample.pdf_data.report_keywords and SAR_DICT_KEY in sample.pdf_data.report_keywords + + sec_level_sars = SARTransformer._parse_sars_from_security_level_list(cert.security_level) + + if st_keywords_may_have_sars(cert): + st_dict: dict = cast(Dict, cert.pdf_data.st_keywords) + st_sars = SARTransformer._parse_sar_dict(st_dict[SAR_DICT_KEY], cert.dgst) + else: + st_sars = set() + + if report_keywords_may_have_sars(cert): + report_dict: dict = cast(Dict, cert.pdf_data.report_keywords) + report_sars = SARTransformer._parse_sar_dict(report_dict[SAR_DICT_KEY], cert.dgst) + else: + report_sars = set() + + return sec_level_sars, st_sars, report_sars + + @staticmethod + def _resolve_candidate_conflicts( + sec_level_candidates: set[SAR], st_candidates: set[SAR], report_candidates: set[SAR], cert_dgst: str + ) -> set[SAR] | None: + final_candidates: dict[str, SAR] = {x.family: x for x in sec_level_candidates} + """ + Given three parameters (SAR candidates from csv scan, ST and cert. report), builds final list of SARs in cert. + This is done as follows: + - All SARs from security level are added first + - Non-conflicting SARs from security target are added as well + - Non-conflicting SARs from ceritifcation report are added at last + + Note: Conflict means an attempt to add SAR of family (but different level) that is already present in the set. + + :return Set | None: Returns set of SARs or None if empty + """ + for candidate in st_candidates: + if candidate.family in final_candidates and candidate != final_candidates[candidate.family]: + logger.debug( + f"Cert {cert_dgst} SAR conflict: Attempting to add {candidate} from ST, conflicts with {final_candidates[candidate.family]}" + ) + else: + final_candidates[candidate.family] = candidate + + for candidate in report_candidates: + if candidate.family in final_candidates and candidate != final_candidates[candidate.family]: + logger.debug( + f"Cert {cert_dgst} SAR conflict: Attempting to add {candidate} from cert. report, conflicts with {final_candidates[candidate.family]}" + ) + else: + final_candidates[candidate.family] = candidate + + return set(final_candidates.values()) if final_candidates else None + + @staticmethod + def _parse_sar_dict(dct: dict[str, dict[str, int]], dgst: str) -> set[SAR]: + """ + Accepts st_keywords or report_keywords dictionary. Will reconstruct SAR objects from it. Each SAR family can + appear multiple times in the dictionary (due to conflicts) with different levels. Iterated item will replace + existing record if: + - it has higher level than than the current SAR + + Only SARs with recovered level are considered, e.g. ASE_REQ.2 is valid string while ASE_REQ is not. + + :param dct: _description_ + :param dgst: DIgest of the processed certificate. + :return: _description_ + """ + sars: dict[str, tuple[SAR, int]] = dict() + for sar_class, class_matches in dct.items(): + for sar_string, n_occurences in class_matches.items(): + try: + candidate = SAR.from_string(sar_string) + except ValueError as e: + logger.debug(f"Badly formatted SAR string {sar_string}, skipping: {e}") + continue + + if candidate.family in sars: + logger.debug( + f"Cert {dgst} Attempting to add {candidate} while {sars[candidate.family]} already in SARS" + ) + + if (candidate.family not in sars) or ( + candidate.family in sars and candidate.level > sars[candidate.family][0].level + ): + sars[candidate.family] = (candidate, n_occurences) + return {x[0] for x in sars.values()} if sars else set() + + @staticmethod + def _parse_sars_from_security_level_list(lst: Iterable[str]) -> set[SAR]: + sars = set() + for element in lst: + try: + sars.add(SAR.from_string(element)) + except ValueError: + continue + return sars diff --git a/src/sec_certs/model/transitive_vulnerability_finder.py b/src/sec_certs/model/transitive_vulnerability_finder.py new file mode 100644 index 00000000..1d4c8243 --- /dev/null +++ b/src/sec_certs/model/transitive_vulnerability_finder.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import logging +from collections import Counter +from dataclasses import dataclass, field +from enum import Enum +from typing import Callable, Dict, Optional, Set, TypeVar + +from sec_certs.sample.certificate import Certificate, References +from sec_certs.serialization.json import ComplexSerializableType + +CertSubType = TypeVar("CertSubType", bound=Certificate) +IDLookupFunc = Callable[[CertSubType], str] +ReferenceLookupFunc = Callable[[CertSubType], References] + + +class ReferenceType(Enum): + DIRECT = "direct" + INDIRECT = "indirect" + + +@dataclass +class TransitiveCVEs(ComplexSerializableType): + direct_transitive_cves: set[str] | None = field(default=None) + indirect_transitive_cves: set[str] | None = field(default=None) + + +Certificates = Dict[str, CertSubType] +Vulnerabilities = Dict[str, Dict[str, Optional[Set[str]]]] + + +class TransitiveVulnerabilityFinder: + """ + The class assigns vulnerabilities to each certificate instance caused by references among certificate instances. + Adheres to sklearn BaseEstimator interface. + """ + + def __init__(self, id_func: IDLookupFunc): + self.vulnerabilities: Vulnerabilities = {} + self.certificates: Certificates = {} + self._cert_id_counter: Counter = Counter() + self._fitted = False + self._id_func: IDLookupFunc = id_func + + def _clear_state(self) -> None: + self.vulnerabilities = {} + self.certificates = {} + self._cert_id_counter = Counter() + + def _fill_dataset_cert_ids_counter(self) -> None: + self._cert_id_counter = Counter([self._id_func(x) for x in self.certificates.values()]) + + def _get_cert_transitive_cves( + self, cert: CertSubType, reference_type: ReferenceType, ref_func: ReferenceLookupFunc + ) -> set[str] | None: + + references = ( + ref_func(cert).directly_referenced_by + if reference_type == ReferenceType.DIRECT + else ref_func(cert).indirectly_referenced_by + ) + + if not references: + return None + + vulnerabilities = set() + + for cert_id in references: + if self._cert_id_counter[cert_id] != 1: + continue + + for cert in self.certificates.values(): + cves = cert.heuristics.related_cves + if self._id_func(cert) == cert_id and cves: + vulnerabilities.update(cves) + + return vulnerabilities if vulnerabilities else None + + def fit(self, certificates: Certificates, ref_func: ReferenceLookupFunc) -> Vulnerabilities: + """ + Method assigns each certificate vulnerabilities caused by references among certificates + + :param Certificates certificates: Dictionary of certificates with digests + :return Vulnerabilities: Dictionary of vulnerabilities of certificate instances + """ + self._clear_state() + self.certificates = certificates + self._fill_dataset_cert_ids_counter() + + thrown_away_cert_counter = 0 + + for cert in self.certificates.values(): + cert_id = self._id_func(cert) + + if not cert_id: + continue + if self._cert_id_counter[cert_id] > 1: + thrown_away_cert_counter += 1 + continue + + self.vulnerabilities[cert.dgst] = dict() + self.vulnerabilities[cert.dgst][ReferenceType.DIRECT.value] = self._get_cert_transitive_cves( + cert, ReferenceType.DIRECT, ref_func + ) + self.vulnerabilities[cert.dgst][ReferenceType.INDIRECT.value] = self._get_cert_transitive_cves( + cert, ReferenceType.INDIRECT, ref_func + ) + + if thrown_away_cert_counter > 0: + logging.warning("There were total of %s certificates skipped due to duplicity", thrown_away_cert_counter) + + self._fitted = True + + return self.vulnerabilities + + def predict_single_cert(self, dgst: str) -> TransitiveCVEs: + """ + Method returns vulnerabilities for certificate digest + + :param str dgst: Digest of certificate + :return TransitiveCVE: TransitiveCVE object of certificate + """ + if not self._fitted: + raise ValueError("Finder not yet fitted") + + if not self.vulnerabilities.get(dgst): + return TransitiveCVEs(direct_transitive_cves=None, indirect_transitive_cves=None) + + return TransitiveCVEs( + self.vulnerabilities[dgst][ReferenceType.DIRECT.value], + self.vulnerabilities[dgst][ReferenceType.INDIRECT.value], + ) + + def predict(self, dgst_list: list[str]) -> dict[str, TransitiveCVEs]: + """ + Method returns vulnerabilities for a list of certificate digests + + :param List[str] dgst_list: list of certificate digests + :return Dict[str, TransitiveCVE]: Dictionary of TransitiveCVE objects for specified certificate digests + """ + if not self._fitted: + raise ValueError("Finder not yet fitted") + + cert_vulnerabilities = {} + + for dgst in dgst_list: + cert_vulnerabilities[dgst] = self.predict_single_cert(dgst) + + return cert_vulnerabilities diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml new file mode 100644 index 00000000..87589d72 --- /dev/null +++ b/src/sec_certs/rules.yaml @@ -0,0 +1,1186 @@ +--- + +##### +# Common Criteria certificate IDs, grouped by scheme (Alpha-2 ISO country code). +##### +cc_cert_id: + DE: + - "BSI-DSZ-CC-[0-9]+?-[0-9]+" + - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+-[0-9]+" + - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+" + - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(-[0-9][0-9][0-9][0-9])*" # German BSI (number + version + year or without year) + - "BSI-DSZ-CC-[0-9]+-[0-9][0-9][0-9][0-9]" # German BSI (number + year, no version) + - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(?!-)" # German BSI (number + version but no year => no - after version) + # - "BSI-DSZ-CC-[0-9]+" # Maybe? + FR: + - "ANSS[Ii](?:-|-CC-|-CC )[0-9]{4}/[0-9]+(v[1-9])?" # French + - "ANSS[Ii]-CC[ -][0-9]{4}[/-_][0-9][0-9]+(?!-M|-S|-R)" # French (/two or more digits then NOT -M or -S) + - "ANSS[Ii]-CC[ -][0-9]{4}[/-_][0-9]+(?:v[0-9])?[_/-][MSR][0-9]+" # French, maintenance or surveillance report (ANSSI-CC-2014_46_M01) + # 'ANSSI-CC-CER-F-.+?', # French + - "DCSS[Ii]-[0-9]+/[0-9]+" # French (DCSSI-2009/07) + - "Certification Report [0-9]+/[0-9]+" # French or Australia! Solved because we limit ourselves to scheme when doing heuristics. + - "Rapport de certification [0-9]+/[0-9]+" # French + NL: + - "NSCIB-CC-[0-9][0-9][0-9][0-9].+?" # Netherlands + - "NSCIB-CC-[0-9][0-9][0-9][0-9][0-9]*-CR" # Netherlands + - "NSCIB-CC-[0-9][0-9]-[0-9]+?-CR[0-9]+?" # Netherlands + - "NSCIB-CC-[0-9][0-9]-[0-9]+(-CR[0-9]+)*" # Netherlands (old number NSCIB-CC-05-6609 or NSCIB-CC-05-6609-CR) + - "NSCIB-CC-[0-9]+-CR[0-9]*" # Netherlands (new number NSCIB-CC-111441-CR NSCIB-CC-111441-CR1) + - "NSCIB-CC-[0-9]+-MA[0-9]*" # Netherlands (new number NSCIB-CC-222073-MA NSCIB-CC-200716-MA2) + - "NSCIB-CC-[0-9][0-9]-[0-9]+" # Netherlands (old number NSCIB-CC-05-6609) + - "NSCIB-CC-[0-9][0-9]-[0-9]+-CR[0-9]+" # Netherlands (NSCIB-CC-year2digits-number-CR) + "NO": + - "SERTIT-[0-9]+" # Norway + US: + - "CCEVS-VR-(?:CC-|VID)?[0-9]+-[0-9]+[a-z]?(?:-[0-9]+)?" # US NSA (CCEVS-VR-10884-2018 CCEVS-VR-VID10877-2018) + CA: + # '[0-9][0-9\-]+?-CR', # Canada + - "[0-9][0-9][0-9]-[347]-[0-9][0-9][0-9]?(?:-CR|P)?" # Canada xxx-{347}-xxx (383-4-438, 383-4-82-CR, 383-4-422P) + - "[0-9][0-9][0-9][ -](?:EWA|LSS|CCS)(?:[ -]20[0-9][0-9])?" # Canada (522-EWA-2020, 524 LSS 2020, 503-LSS) + - "[0-9][0-9][0-9](?:%20|-)(?:EWA|LSS|CCS)(?:%20|-)(?:20[0-9][0-9]%20|)CR%20v[0-9]\\.[0-9]" # Canada filename with space (518-LSS%20CR%20v1.0) + UK: + - "CRP[0-9]+[A-Z]?" # UK CESG + - "CERTIFICATION REPORT No. P[0-9]+[A-Z]?" # UK CESG + ES: + - "20[0-9][0-9][-‐][0-9]+[-‐]INF[-‐][0-9]+([-‐]?[ -‐](?:V|v)[0-9]+)?" # Spain ("2006-4-INF-98 v2" or "2006-4-INF-98-v2" or "2020-34-INF-3784- v1") + KR: + # Korea + # XXX: Do not use KECS-CR as those refer to the certificate report and do not represent the certificate id. + # - "KECS[-‐]CR[-‐][0-9]+[-‐][0-9]+" # Korea KECS-CR-20-61 + - "KECS[-‐](?:ISIS|NISS|CISS)[-‐][0-9]+[-‐][0-9]{4}" # Korea KECS-ISIS-1234-2011 + JP: + - "(?:CRP|ACR)-C[0-9]+-[0-9]+" # Japan (CRP-C0595-01 ACR-C0417-03) + - "JISEC-CC-CRP-C[0-9]+-[0-9]+-[0-9]+" # Japan (JISEC-CC-CRP-C0689-01-2020) + - "Certification No. [cC][0-9]+" # Japan (Certification No. C0090) + MY: + - "ISCB-[0-9]+-(?:RPT|FRM)-[CM][0-9]+[A-Z]?-(?:CR|AMR)(?:-[0-9])?-[vV][0-9](?:\\.[0-9])?[a-z]?" # Malaysia (ISCB-3-RPT-C092-CR-v1, ISCB-3-RPT-C068-CR-1-v1) + IT: + - "OCSI/CERT/.+?" # Italy + - "OCSI/CERT/.+?/20[0-9]+(?:\\w|/RC)" # Italy (OCSI/CERT/ATS/01/2018/RC) + TR: + - "[0-9\\.]+?/TSE-CCCS-[0-9]+" # Turkish CCCS (21.0.0sc/TSE-CCCS-75) + - "(?:[0-9]{1,2}\\.){2}[0-9]{1,2}/[0-9]{1,4}-[0-9]{3}" # 21.0.01/13-028 + SE: + - "CSEC ?[0-9]{6,7}" # Sweden (CSEC2019015) + IN: + # India (IC3S/DEL01/VALIANT/EAL1/0317/0007/CR STQC/CC/14-15/12/ETR/0017 IC3S/MUM01/CISCO/cPP/0119/0016/CR) + # will miss STQC/CC/14-15/12/ETR/0017 + - "(?:IC3S|STQC/CC)/[^ ]+? ?/CR" + SG: + - "CSA_CC_[0-9]+" # Singapore (CSA_CC_19001) + AU: + # Australia (EFS-T048 ETR 1.0, EFS-T056-ETR 1.0, DXC-EFC-T092-ETR 1.0) + # XXX: Do not use Australian ETR numbers, they are not certificate id. + # - "(?:EFS|EFT|DXC-EFC)-T[0-9]+(?: |-)ETR [0-9]+.[0-9]+" + - "Certificate Number: [0-9]{1,4}/[0-9]{1,4}" + - "Certification Report [0-9]+/[0-9]+" + +##### +# Common Criteria protection profile IDs, grouped by certification body (e.g. BSI) +##### +cc_protection_profile_id: + BSI: + - "BSI-(?:CC[-_]|)PP[-_]*.+?" + - "BSI-CCPP-.+?" + ANSSI: + - "ANSSI-CC-PP.+?" + KECS: + - "KECS-PP-[0-9]+-[0-9]+" + other: + - "PP-SSCD.+?" + - "PP_DBMS_.+?" + - "WBIS_V[0-9]\\.[0-9]" + - "EHCT_V.+?" + +##### +# Common Criteria security level (EAL or ITSEC). +##### +cc_security_level: + EAL: + - "EAL[ ]*[0-9+]+?" + - "EAL[ ]*[0-9] augmented" + ITSEC: + - "ITSEC[ ]*E[1-9]*.+?" + +##### +# Common Criteria security assurance requirement (SAR) code, grouped by class (e.g. ACE, ACM, ...). +##### +cc_sar: + ACE: + - "ACE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + ACM: + - "ACM(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + ACO: + - "ACO(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + ADO: + - "ADO(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + ADV: + - "ADV(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + AGD: + - "AGD(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + ALC: + - "ALC(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + ATE: + - "ATE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + AVA: + - "AVA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + AMA: + - "AMA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + APE: + - "APE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + ASE: + - "ASE(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + +##### +# Common Criteria security functional requirement (SFR) code, grouped by class (e.g. FAU, FCO, ...). +##### +cc_sfr: + FAU: + - "FAU(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FCO: + - "FCO(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FCS: + - "FCS(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FDP: + - "FDP(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FIA: + - "FIA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FMT: + - "FMT(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FPR: + - "FPR(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FPT: + - "FPT(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FRU: + - "FRU(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FTA: + - "FTA(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + FTP: + - "FTP(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}" + +##### +# Common Criteria claim code, grouped by class (e.g. D, T, ...). +##### +cc_claims: + D: + - "D\\.[\\._\\-A-Z]+?" + O: + - "O\\.[\\._\\-A-Z]+?" + T: + - "T\\.[\\._\\-A-Z]+?" + A: + - "A\\.[\\._\\-A-Z]+?" + R: + - "R\\.[\\._\\-A-Z]+?" + OT: + - "OT\\.[\\._\\-A-Z]+?" + OP: + - "OP\\.[\\._\\-A-Z]+?" + OE: + - "OE\\.[\\._\\-A-Z]+?" + SA: + - "SA\\.[\\._\\-A-Z]+?" + OSP: + - "OSP\\.[\\._\\-A-Z]+?" + +##### +# A generic vendor of a product, mostly has smartcard/secure hardware vendors or other large vendors of certified products. +##### +vendor: + NXP: + - "NXP( Semiconductors)?( N\\.V\\.)?" + Infineon: + - "Infineon( Technologies)?( AG)?" + Samsung: + - "Samsung" + STMicroelectronics: + - "STMicroelectronics|STM|STMicro( N\\.V\\.)?" + Feitian: + - "Feitian( Technologies)?( Co.)?" + Gemalto: + - "Gemalto( N\\.V\\.)?" + Gemplus: + - "Gemplus( International)?( SA)?" + Axalto: + - "Axalto" + Thales: + - "Thales( Group)?( SA)?" + Oberthur: + - "(Oberthur|OBERTHUR)( Technologies| Card Systems)?" + Idemia: + - "Idemia|IDEMIA" + Sagem: + - "Sagem|SAGEM" + Morpho: + - "Morpho( Systèmes)?" + GD: + - "G&D|G\\+D|Giesecke\\+Devrient|Giesecke & Devrient" + Philips: + - "(Koninklijke )?Philips( N\\.V\\.)?" + Qualcomm: + - "Qualcomm" + Broadcom: + - "Broadcom( Inc.)?" + Huawei: + - "Huawei( Technologies)?( Co.)?" + Microsoft: + - "Microsoft( Corporation)?" + Cisco: + - "Cisco( Systems)?(, Inc.)?" + +##### +# Common Criteria evaluation facility, mostly from https://www.commoncriteriaportal.org/labs/, grouped roughly by facility. +##### +eval_facility: + Serma: + - "Serma Technologies|SERMA|Serma Safety & Security" + Thales: + - "THALES - CEACI|THALES/CNES" + Riscure: + - "Riscure" + SGS: + - "SGS" + - "SGS Bright[sS]ight" # SGS acquired BrightSight + BrightSight: + - "Bright[sS]ight" + Applus: + - "Applus Laboratories" + TUV: + - "(tuvit|TÜViT|TUViT|TÜV Informationstechnik|TUV Informationstechnik)" + CESTI: + - "CESTI" + DXC: + - "DXC Technology" + Teron: + - "Teron Labs" + EWA: + - "(EWA|EWA-Canada)" + Lightship: + - "Lightship Security" + AMOSSYS: + - "AMOSSYS" + CEA-LETI: + - "(CEA - LETI|CEA/LETI|CEA-LETI)" + OPPIDA: + - "OPPIDA" + TrustedLabs: + - "Trusted Labs" + atsec: + - "atsec" + DeutscheTelekom: + - "Deutsche Telekom Security" + DFKI: + - "(Deutsches Forschungszentrum für künstliche Intelligenz|dfki|DFKI)" + MTG: + - "MTG AG" + secuvera: + - "secuvera" + SRC: + - "SRC Security Research & Consulting" + Acucert: + - "Acucert Labs" + ERTL: + - "Common Criteria Test Laboratory,? ERTL" + ETDC: + - "Common Criteria Test Laboratory,? ETDC" + CCLab: + - "CCLab Software Laboratory" + Deeplab: + - "Deeplab" + IMQLPS: + - "IMQ/LPS" + LVSLeonardo: + - "LVS Leonardo" + LVSTechnisBlu: + - "LVS Technis Blu" + ECSEC: + - "ECSEC Laboratory" + ITSC: + - "Information Technology Security Center" + Acumen: + - "Acumen Security" + BoozAllenHamilton: + - "Booz Allen Hamilton" + Gossamer: + - "Gossamer Security" + Leidos: + - "Leidos" + UL: + - "UL Verification Services" + BEAM: + - "BEAM Teknoloji" + Certby: + - "Certby Lab" + DEKRA: + - "DEKRA Testing and Certification" + STMITSEF: + - "STM ITSEF" + TUBITAK-BILGEM: + - "TÜBİTAK BİLGEM" + Combitech: + - "Combitech AB" + Intertek: + - "Intertek" + Clover: + - "Clover Technologies" + LAYAKK: + - "LAYAKK SEGURIDAD INFORMATICA" + An: + - "An Security" + TSystems: + - "T-Systems International" + KISA: + - "KISA" + KOIST: + - "KOIST" + KSEL: + - "KSEL" + KOSYAS: + - "KOSYAS" + KTR: + - "KTR" + KTC: + - "KTC" + TTA: + - "TTA" + ADS: + - "Advanced Data Security" + Nemko: + - "Nemko System Sikkerhet" + Norconsult: + - "Norconsult( AS)?" + Secura: + - "Secura" + BAE: + - "BAE Applied Intelligence" + +##### +# Symmetric crypto primitive (e.g. a block or stream cipher), grouped by competition/standardization effort. +##### +symmetric_crypto: + AES_competition: + AES: + - "AES-?(?P128|192|256)?" + Rijndael: + - "Rijndael" + Twofish: + - "Twofish" + Serpent: + - "Serpent" + MARS: + - "MARS" + HPC: + - "HPC" + FROG: + - "FROG" + CAST: + - "CAST-?(?P128|160|192|224|256|5)?" + RC: + - "RC(?P[2456])" + CRYPTON: + - "CRYPTON" + DEAL: + - "DEAL" + E2: + - "E2" + LOKI97: + - "LOKI97" + MAGENTA: + - "MAGENTA" + SAFER: + - "SAFER\\+" + DES: + DES: + - "DE[SA]" + 3DES: + - "([3T]|Triple)-?DE[SA]" + Lucifer: + - "Lucifer" + djb: + ChaCha: + - "[XH]?ChaCha(?P20|8)?" + Salsa: + - "[XH]?Salsa(?P20(/8|/12)?)?" + Poly: + - "Poly1305" + LWC_competition: + ASCON: + - "(ASCON|Ascon)" + Elephant: + - "Elephant" + GIFT: + - "GIFT(-COFB)?" + Grain: + - "Grain128(-AEAD)?" + ISAP: + - "ISAP" + PhotonBeetle: + - "Photon-?Beetle" + Romulus: + - "Romulus" + Sparkle: + - "Sparkle" + TinyJambu: + - "TinyJambu" + Xoodyak: + - "Xoodyak" + Gimli: + - "Gimli" + eSTREAM: + HC: + - "HC-[0-9]{3}" + Rabbit: + - "Rabbit" + SOSEMANUK: + - "SOSEMANUK" + MICKEY: + - "MICKEY(-128)?" + Trivium: + - "Trivium" + CAESAR: + ACORN: + - "ACORN" + AEGIS: + - "AEGIS(-128)?" + Deoxys: + - "Deoxys(-2)?" + COLM: + - "COLM" + miscellaneous: + IDEA: + - "IDEA" + Blowfish: + - "Blowfish" + Camellia: + - "Camellia" + ARIA: + - "ARIA" + SM4: + - "SM4" + GOST: + - "GOST 28147-89" + - "Magma" + - "Kuznyechik" + SEED: + - "SEED" + Skipjack: + - "Skipjack" + Skinny: + - "Skinny|SKINNY" + constructions: + MAC: + - "HMAC" + - "HMAC-[A-Z]+?-(?:160|224|256|384|512)" + - "KMAC" + - "CMAC|CBC-MAC" + +##### +# Asymmetric crypto primitive, grouped by type (RSA, ECC, FF). +##### +asymmetric_crypto: + RSA: + - "RSA[- ]?(?P512|768|1024|1280|1536|2048|3072|4096|8192)" + - "RSASSAPKCS1-[Vv]1_5" + - "RSA-?(OAEP|PSS|CRT)" + ECC: + ECDH: + - "ECDHE?" + ECDSA: + - "ECDSA" + EdDSA: + - "EdDSA" + ECIES: + - "ECIES" + ECC: + - "ECC" + FF: + DH: + - "Diffie-Hellman|DHE?" + DSA: + - "DSA" + +##### +# Post-quantum crypto primitive, grouped by primitive, from NIST-PQC. +##### +pq_crypto: + ClassicMcEliece: + - "Classic[ -]McEliece" + Kyber: + - "(CRYSTALS-)?(Kyber|KYBER)" + NTRU: + - "NTRU" + NTRUPrime: + - "NTRU[ -]?Prime" + Saber: + - "Saber|SABER" + Dilithium: + - "(CRYSTALS-)?(Dilithium|DILITHIUM)" + Falcon: + - "Falcon|FALCON" + Rainbow: + - "Rainbow" + UOV: + - "UOV" + BIKE: + - "BIKE" + Frodo: + - "Frodo(KEM)?" + HQC: + - "HQC" + SIKE: + - "SIKE" + SIDH: + - "SIDH" + GeMSS: + - "GeMSS" + Picnic: + - "Picnic" + SPHINCS: + - "SPHINCS\\+" + +##### +# Hash-function, grouped by hash-function class (SHA, MD) or competition (PHC). +##### +hash_function: + SHA: + SHA1: + - "SHA-?1" + SHA2: + - "SHA-?(?P160|224|256|384|512)" + - "SHA-?2" + SHA3: + - "SHA-?3(-[0-9]{3})?" + Keccak: + - "Keccak" + SHAKE: + - "SHAKE[0-9]{3}" + Groestl: + - "(Groestl|Grøstl)" + BLAKE: + - "(Blake|BLAKE)[23][sbX]?" + JH: + - "JH" + Skein: + - "Skein" + MD: + MD4: + - "MD4" + MD5: + - "MD5" + MD6: + - "MD6" + RIPEMD: + - "RIPEMD(-?[0-9]{3})?" + Streebog: + - "Streebog" + Whirpool: + - "Whirpool" + PHC_competition: + Argon: + - "Argon2?[id]?" + battcrypt: + - "battcrypt" + Catena: + - "Catena" + Lyra2: + - "Lyra2" + Makwa: + - "Makwa" + POMELO: + - "POMELO" + Pufferfish: + - "Pufferfish" + yescrypt: + - "yescrypt" + bcrypt: + - "bcrypt" + scrypt: + - "scrypt" + PBKDF: + - "PBKDF[12]?" + +##### +# General cryptographic scheme. +##### +crypto_scheme: + MAC: + - "MAC" + KEM: + - "KEM" + PKE: + - "PKE" + KEX: + - "KEX|Key [eE]xchange" + KA: + - "KA|Key [aA]greement" + PAKE: + - "PAKE" + AEAD: + - "AEAD" + +##### +# General cryptographic protocol. +##### +crypto_protocol: + SSH: + - "SSH" + TLS: + SSL: + - "SSL( ?v?[123]\\.0)?" + TLS: + - "TLS( ?v?1\\.[0123])?" + DTLS: + - "DTLS( ?v?1\\.[0123])?" + PACE: + - "PACE" + IKE: + - "IKE(v[12])?" + IPsec: + - "IPsec" + VPN: + - "VPN" + PGP: + - "PGP" + +##### +# Random number generator. +##### +randomness: + DUAL_EC: + - "DUAL_EC_DRBG" + TRNG: + - "D?TRNG" + PRNG: + - "PRNG" + - "DRBG" + RNG: + - "RN[GD]" + - "RBG" + +##### +# Block cipher mode. +##### +cipher_mode: + ECB: + - "ECB" + CBC: + - "CBC" + CTR: + - "CTR" + CFB: + - "CFB" + OFB: + - "OFB" + GCM: + - "GCM" + SIV: + - "SIV" + XTR: + - "XTR" + CCM: + - "CCM" + LRW: + - "LRW" + XEX: + - "XEX" + XTS: + - "XTS" + +##### +# An elliptic curve, grouped by standardization body (e.g. NIST, Brainpool). +# Note that multiple curve names may correspond to the same curve. +##### +ecc_curve: + NIST: + - "(?:Curve |curve |)P-(192|224|256|384|521)" + - "(?:ansit|ansip|ANSIP|ANSIT)[0-9]+?[rk][12]" + - "(NIST)? ?[PBK]-[0-9]{3}" + - "(?:secp|sect|SECP|SECT)[0-9]+?[rk][12]" + - "prime[0-9]{3}v[123]" + - "c2[pto]nb[0-9]{3}[vw][123]" + Brainpool: + - "(?:brainpool|BRAINPOOL)P[0-9]{3}[rkt][12]" + ANSSI: + - "(?:anssi|ANSSI)?[ ]*FRP[0-9]+?v1" + NUMS: + - "numsp[0-9]{3}[td]1" + Curve: + - "Curve(25519|1174|4417|22103|67254|383187|41417)" + Edwards: + - "Ed(25519|448)" + ssc: + - "ssc-(160|192|224|256|288|320|384|512)" + Tweedle: + - "Tweedle(dee|dum)" + Pasta: + - "(Pallas|Vesta)" + JubJub: + - "JubJub" + BLS: + - "BLS(12|24)-[0-9]{3}" + BN: + - "bn[0-9]{3}" + +##### +# A cryptographic engine. +##### +crypto_engine: + TORNADO: + - "TORNADO" + SmartMX: + - "SmartMX2?" + NexCrypt: + - "NexCrypt" + +##### +# TLS cipher suite name. +##### +tls_cipher_suite: + TLS: + - "TLS(_[A-Z0-9]+){1,3}_WITH(_[A-Z0-9]+){2,4}" + +##### +# A cryptographic library, grouped by rough library category. +##### +crypto_library: + Neslib: + - "(?:NesLib|NESLIB) [v]*[0-9\\.]+" + AT1: + - "AT1 Secure .{1,30}? Library [v]*[0-9\\.]+" + - "AT1 Secure RSA/ECC/SHA library" + Generic: + - "Crypto Library [v]*[0-9\\.]+" + AtmelToolbox: + - "(ATMEL|Atmel) Toolbox [0-9\\.]+" + Infineon: + - "v1\\.02\\.013" # Infineon's ROCA-vulnerable library + OpenSSL: + - "OpenSSL" + LibreSSL: + - "LibreSSL" + BoringSSL: + - "BoringSSL" + MatrixSSL: + - "MatrixSSL" + Nettle: + - "Nettle" + GnuTLS: + - "GnuTLS" + libtomcrypt: + - "libtomcrypt" + BearSSL: + - "BearSSL" + Botan: + - "Botan" + Crypto++: + - "Crypto\\+\\+" + wolfSSL: + - "wolfSSL" + mbedTLS: + - "[mM]bedTLS" + s2n: + - "(Amazon )?s2n" + NSS: + - "NSS" + libgcrypt: + - "libgcrypt" + BouncyCastle: + - "BouncyCastle" + cryptlib: + - "cryptlib" + NaCl: + - "NaCl" + libsodium: + - "libsodium" + libsecp256k1: + - "libsecp256k1" + +##### +# A vulnerability idenfitier or name (e.g. CVE-... but also Minerva, ROCA). +##### +vulnerability: + CVE: + - "CVE-[0-9]+?-[0-9]+?" + CWE: + - "CWE-[0-9]+?" + ROCA: + - "ROCA" + Minerva: + - "Minerva" + TPM-Fail: + - "TPM[\\.-]Fail" + +##### +# A side-channel analysis related term, grouped into SCA, FI and other. +##### +side_channel_analysis: + SCA: + - "Leak-Inherent" + - "[Pp]hysical [Pp]robing" + - "[Ss]ide.channels?" + - "SPA|DPA" + - "[tT]iming [aA]ttacks?" + - "[Tt]emplate [aA]ttacks?" + - "[Pp]rofiled [aA]ttacks?" + - "[Cc]lustering [aA]ttacks?" + - "t-test|TVLA" + FI: + - "[pP]hysical [tT]ampering" + - "[Mm]alfunction" + - "DFA|SIFA" + - "[Ff]+ault [iI]nduction" + - "[Ff]+ault [iI]njection" + other: + - "[Dd]eep[ -][lL]earning" + - "[Cc]old [bB]oot" + - "[Rr]ow-?hammer" + - "[Rr]everse [eE]ngineering" + - "[Ll]attice [aA]ttacks?" + - "[Oo]racle [aA]ttacks?" + - "[Bb]leichenbacher [aA]ttacks?" + - "[Bb]ellcore [aA]ttacks?" + - "JIL(-(AAPS|COMP|AM|AAPHD|AMHD))?" + - "JHAS" + +##### +# A term used in the certification process. +##### +certification_process: + OutOfScope: + - "[oO]ut of [sS]cope" + - "[\\.\\(].{0,100}?.[oO]ut of [sS]cope..{0,100}?[\\.\\)]" + - ".{0,100}[oO]ut of [sS]cope.{0,100}" + ConfidentialDocument: + - ".{0,100}confidential document.{0,100}" + SecurityFunction: + - "[sS]ecurity [fF]unction SF\\.[a-zA-Z0-9_]" + +##### +# A technical report id, grouped by standardization body (e.g. BSI). +##### +technical_report_id: + BSI: + - "BSI[ ]*TR-[0-9]+?(?:-[0-9]+?|)" + - "BSI [0-9]+?" # German BSI document containing list of issued certificates in some period + +##### +# A device model, grouped by manufacturer and subgroups into particular model families. +##### +device_model: + G87: + - "G87-.+?" + ATMEL: + - "ATMEL AT.+?" + STM: + STM32: + - "STM32[FGLHW][0-7][0-9]{1,2}[FGKTSCRVZI][468BCDEFGHI][PHUTY][67]" + Infineon: + SLE: + - "SLE[0-9]{2}[A-Z]{3}[0-9]{1-4}[A-Z]{1-3}" + +##### +# A Trusted Execution Environment, grouped by manufacturer (e.g. Intel, ARM, ...). +##### +tee_name: + Intel: + - "(Intel )?SGX" + ARM: + - "(ARM )?TrustZone" + - "(ARM )?(Realm Management Extension|Confidential Compute Architecture)" + AMD: + - "(AMD )?(PSP|Platform Security Processor)" + - "(AMD )?(SEV|Secure Encrypted Virtualization)" + IBM: + - "(IBM )?(SSC|Secure Service Container)" + - "(IBM )?(SE|Secure Execution)" + other: + - "Cloud Link TEE" + - "iOS Secure Enclave" + - "iTrustee" + - "Trusty" + - "OPTEE" + - "QTEE" + - "TEEgris" + - "T6" + - "Kinibi" + - "SW TEE" + - "WatchTrust" + - "TEE" + +##### +# An OS name, grouped by OS. +##### +os_name: + STARCOS: + - "STARCOS(?: [0-9\\.]+?|)" + JCOP: + - "JCOP[ ]*[0-9]" + +##### +# CPLC data name.. +##### +cplc_data: + ICFab: + - "IC[ \\.]*Fabricator" + ICType: + - "IC[ \\.]*Type" + ICVersion: + - "IC[ \\.]*Version" + +##### +# An elementary data group. +##### +ic_data_group: + EF: + - "EF\\.DG[1-9][0-6]?" + - "EF\\.COM" + - "EF\\.CardAccess" + - "EF\\.SOD" + - "EF\\.ChipSecurity" + +##### +# Standard ID, grouped by standardization body (e.g. FIPS, NIST, ISO). +##### +standard_id: + FIPS: + - "FIPS ?(?:PUB )?[0-9]+(-[0-9]+)?" + NIST: + - "(NIST )?SP [0-9]+-[0-9]+?[a-zA-Z]?" + PKCS: + - "PKCS[ #]*[1-9]+" + BSI: + - "BSI-AIS[ ]*[0-9]+?" + - "AIS[ ]*[0-9]+?" + RFC: + - "RFC[ -]?[0-9]+?" + ISO: + - "ISO/IEC[ ]*[0-9]+[-]*[0-9]*" + - "ISO/IEC[ ]*[0-9]+:[ 0-9]+" + - "ISO/IEC[ ]*[0-9]+" + ICAO: + - "ICAO(?:-SAC|)" + X509: + - "[Xx]\\.509" + SCP: + - "(?:SCP|scp)[ ']*[0-9][0-9]" + CC: + - "CC[I]*MB-20[0-9]+?-[0-9]+?-[0-9]+?" # Common Criteria methodology + - "CCIMB-9[0-9]-[0-9]+?" # Common Criteria methodology old + +##### +# JavaCard version identifier. +##### +javacard_version: + JavaCard: + - "(?:Java Card|JavaCard) [2-3]\\.[0-9](?:\\.[0-9]|)" + - "JC[2-3]\\.[0-9](?:\\.[0-9]|)" + - "(?:Java Card|JavaCard) \\(version [2-3]\\.[0-9](?:\\.[0-9]|)\\)" + GlobalPlatform: + - "(?:Global Platform|GlobalPlatform) [2-3]\\.[0-9]\\.[0-9]" + - "(?:Global Platform|GlobalPlatform) \\(version [2-3]\\.[0-9]\\.[0-9]\\)" + +##### +# JavaCard API constant, grouped into "ALG", "misc" and "curves". +##### +javacard_api_const: + ALG: + RNG: + - "ALG_(?:PSEUDO_RANDOM|SECURE_RANDOM|TRNG|PRESEEDED_DRBG|FAST|KEYGENERATION)" + DES: + - "ALG_DES_[A-Z_0-9]+" + RSA: + - "ALG_RSA_[A-Z_0-9]+" + DSA: + - "ALG_DSA_[A-Z_0-9]+" + ECDSA: + - "ALG_ECDSA_[A-Z_0-9]+" + AES: + - "ALG_AES_[A-Z_0-9]+" + HMAC: + - "ALG_HMAC_[A-Z_0-9]+" + KOREAN: + - "ALG_KOREAN_[A-Z_0-9]+" + EC: + - "ALG_EC_[A-Z_0-9]+?" + SHA: + - "ALG_SHA_[A-Z_0-9]+" + SHA3: + - "ALG_SHA3_[A-Z_0-9]+" + MD: + - "ALG_MD[A-Z_0-9]+" + RIPEMD: + - "ALG_RIPEMD[A-Z_0-9]+" + ISO3309: + - "ALG_ISO3309_[A-Z_0-9]+" + XDH: + - "ALG_XDH" + SM2: + - "ALG_SM2" + SM3: + - "ALG_SM3" + NULL: + - "ALG_NULL" + misc: + - "SIG_CIPHER_[A-Z_0-9]+" + - "CIPHER_[A-Z_0-9]+" + - "PAD_[A-Z_0-9]+" + - "TYPE_[A-Z_0-9]+" + - "LENGTH_[A-Z_0-9]+" + - "OWNER_PIN[A-Z_0-9]*" + curves: + - "BRAINPOOLP[A-Z_0-9]+(?:R|T)1" + - "ED25519" + - "ED448" + - "FRP256V1" + - "SECP[0-9]*R1" + - "SM2" + - "X25519" + - "X448" + +##### +# JavaCard common package identifiers. +##### +javacard_packages: + java: + - "java\\.[a-z\\.]+" + javacard: + - "javacard\\.[a-z\\.]+" + javacardx: + - "javacardx\\.[a-z\\.]+" + org: + - "org\\.[0-9a-z\\.]+" + uicc: + - "uicc\\.[a-z\\.]+" + com: + - "com\\.[0-9a-z\\.]+" + de: + - "de\\.bsi\\.[a-z\\.]+" + +##### +# FIPS 140 certificate id. +##### +fips_cert_id: + Cert: + - "(?:#[^\\S\\r\\n]?|Cert\\.?(?!.\\s)[^\\S\\r\\n]?|Certificate[^\\S\\r\\n]?)(?P\\d{1,4})(?!\\d)" + +##### +# FIPS 140 security level. +##### +fips_security_level: + Level: + - "[lL]evel (\\d)" + +##### +# FIPS 140 "certlike" string, that needs to get removed from certificate id matches. +##### +fips_certlike: + Certlike: + # --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) --- + # + added (and #id) everywhere + - "HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\\(?(?: |hardware|firmware)*?[\\s(\\[]*?(?:#|cert\\.?|Cert\\.?|Certificate|sample)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:[\\s#]*and[\\s#]*\\d+)?" + # --- same as above, without hw or fw --- + - "HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" + # --- SHS/A - (bits) (method) ((cert #) numbers) --- + - "SH[SA][-– 123]*(?:;|\\/|160|224|256|384|512)?(?:[\\s(\\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\\s,]*?[\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:\\)?\\[#?\\d+\\])?(?:[\\s#]*?and[\\s#]*?\\d+)?" + # --- RSA (bits) (method) ((cert #)) --- + - "RSA(?:[-– ]*(?:;|\\/|512|768|1024|1280|1536|2048|3072|4096|8192)\\s\\(\\[]*?(?:(?:;|\\/|KAT|Verify|PSS|\\s)*?)?[\\s,]*?[\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" + # --- RSA (SSA) (PKCS) (version) (#) --- + - "(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\\s?#?\\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\\s#]*?(\\d{1,4})?" + # --- AES (bits) (method) ((cert #)) --- + - "AES[-– ]*((?: |;|\\/|bit|key|128|192|256|CBC)*(?: |\\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:\\)?[\\s#]*?\\[#?\\d+\\])?(?:[\\s#]*?and[\\s#]*?(\\d+))?" + # --- Diffie Helman (CVL) ((cert #)) --- + - "Diffie[-– ]*Hellman[,\\s(\\[]*?(?:CVL|\\s)*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?[\\s#]*?(\\d{1,4})" + # --- DRBG (bits) (method) (cert #) --- + - "DRBG[ –-]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" + # --- DES (bits) (method) (cert #) + - "DES[ –-]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\\d(?: and \\d)? keying options?))*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)*?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:[\\s#]*?and[\\s#]*?(\\d+))?" + # --- DSA (bits) (method) (cert #) + - "DSA[ –-]*((?:;|\\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})" + # --- platforms (#)+ - this is used in modification history --- + - "[Pp]latforms? #\\d+(?:#\\d+|,| |-|and)*[^\\n]*" + # --- CVL (#) --- + - "CVL[\\s#]*?(\\d{1,4})" + # --- PAA (#) --- + - "PAA[: #]*?\\d{1,4}" + # --- (#) Type --- + - "(?:#|cert\\.?|sample|Cert\\.?|Certificate)[\\s#]*?(\\d+)?\\s*?(?:AES|SHS|SHA|RSA|HMAC|Diffie-Hellman|DRBG|DES|CVL)" + # --- PKCS (#) --- + - "PKCS[\\s]?#?\\d+" + - "PKSC[\\s]?#?\\d+" # typo, #625 + # --- # C and # A (just in case) --- + - "#\\s+?[Cc]\\d+" + - "#\\s+?[Aa]\\d+" + + +##### +# Common Criteria rules. +##### +cc_rules: + - "cc_cert_id" + - "cc_protection_profile_id" + - "cc_security_level" + - "cc_sar" + - "cc_sfr" + - "cc_claims" + - "vendor" + - "eval_facility" + - "symmetric_crypto" + - "asymmetric_crypto" + - "pq_crypto" + - "hash_function" + - "crypto_scheme" + - "crypto_protocol" + - "randomness" + - "cipher_mode" + - "ecc_curve" + - "crypto_engine" + - "tls_cipher_suite" + - "crypto_library" + - "vulnerability" + - "side_channel_analysis" + - "technical_report_id" + - "device_model" + - "tee_name" + - "os_name" + - "cplc_data" + - "ic_data_group" + - "standard_id" + - "javacard_version" + - "javacard_api_const" + - "javacard_packages" + - "certification_process" + + +##### +# FIPS rules. +##### +fips_rules: + - "fips_cert_id" + - "fips_security_level" + - "fips_certlike" + - "vendor" + - "eval_facility" + - "symmetric_crypto" + - "asymmetric_crypto" + - "pq_crypto" + - "hash_function" + - "crypto_scheme" + - "crypto_protocol" + - "randomness" + - "cipher_mode" + - "ecc_curve" + - "crypto_engine" + - "tls_cipher_suite" + - "crypto_library" + - "vulnerability" + - "side_channel_analysis" + - "device_model" + - "tee_name" + - "os_name" + - "cplc_data" + - "ic_data_group" + - "standard_id" + - "javacard_version" + - "javacard_api_const" + - "javacard_packages" + - "certification_process" diff --git a/src/sec_certs/sample/__init__.py b/src/sec_certs/sample/__init__.py new file mode 100644 index 00000000..ecbbd541 --- /dev/null +++ b/src/sec_certs/sample/__init__.py @@ -0,0 +1,33 @@ +"""This package holds mostly data objects of primary interest (Common Criteria, FIPS), or assisting objects +like CPE, CVE, etc. The objects mostly hold data and allow for serialization, but can also perform some basic transformations. +""" + +from sec_certs.sample.cc_certificate_id import CertificateId +from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.cpe import CPE, cached_cpe +from sec_certs.sample.cve import CVE +from sec_certs.sample.fips import FIPSCertificate +from sec_certs.sample.fips_algorithm import FIPSAlgorithm +from sec_certs.sample.fips_iut import IUTEntry, IUTSnapshot +from sec_certs.sample.fips_mip import MIPEntry, MIPSnapshot, MIPStatus +from sec_certs.sample.protection_profile import ProtectionProfile +from sec_certs.sample.sar import SAR + +__all__ = [ + "CertificateId", + "CommonCriteriaMaintenanceUpdate", + "CommonCriteriaCert", + "CPE", + "cached_cpe", + "CVE", + "FIPSCertificate", + "FIPSAlgorithm", + "IUTEntry", + "IUTSnapshot", + "MIPEntry", + "MIPSnapshot", + "MIPStatus", + "ProtectionProfile", + "SAR", +] diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py new file mode 100644 index 00000000..428ca73b --- /dev/null +++ b/src/sec_certs/sample/cc_certificate_id.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass + + +@dataclass(eq=True, frozen=True) +class CertificateId: + """ + A Common Criteria certificate id. + """ + + scheme: str + raw: str + + def _canonical_fr(self) -> str: + new_cert_id = self.clean + rules = [ + "(?:Rapport de certification|Certification Report) ([0-9]+[/-_][0-9]+(?:v[1-9])?(?:[_/-][MSR][0-9]+)?)", + "(?:ANSS[Ii]|DCSSI)(?:-CC)?[- ]([0-9]+[/-_][0-9]+(?:v[1-9])?(?:[_/-][MSR][0-9]+)?)", + "([0-9]+[/-_][0-9]+(?:v[1-9])?(?:[_/-][MSR][0-9]+)?)", + ] + for rule in rules: + if match := re.match(rule, new_cert_id): + return "ANSSI-CC-" + match.group(1).replace("_", "/") + + return new_cert_id + + def _canonical_de(self) -> str: + def extract_parts(bsi_parts: list[str]) -> tuple: + cert_num = None + cert_version = None + cert_year = None + + if len(bsi_parts) > 3: + cert_num = bsi_parts[3] + if len(bsi_parts) > 4: + if bsi_parts[4].startswith("V") or bsi_parts[4].startswith("v"): + cert_version = bsi_parts[4].upper() # get version in uppercase + else: + cert_year = bsi_parts[4] + if len(bsi_parts) > 5: + cert_year = bsi_parts[5] + + return cert_num, cert_version, cert_year + + bsi_parts = self.clean.split("-") + + cert_num, cert_version, cert_year = extract_parts(bsi_parts) + + # reconstruct BSI number again + new_cert_id = "BSI-DSZ-CC" + if cert_num is not None: + new_cert_id += "-" + cert_num + if cert_version is not None: + new_cert_id += "-" + cert_version + if cert_year is not None: + new_cert_id += "-" + cert_year + + return new_cert_id + + def _canonical_es(self) -> str: + cert_id = self.clean + spain_parts = cert_id.split("-") + cert_year = spain_parts[0] + cert_batch = spain_parts[1].lstrip("0") + cert_num = spain_parts[3].lstrip("0") + + if "v" in cert_num: + cert_num = cert_num[: cert_num.find("v")] + if "V" in cert_num: + cert_num = cert_num[: cert_num.find("V")] + + new_cert_id = f"{cert_year}-{cert_batch}-INF-{cert_num.strip()}" # drop version # TODO: Maybe do not drop? + + return new_cert_id + + def _canonical_it(self): + new_cert_id = self.clean + if not new_cert_id.endswith("/RC"): + new_cert_id = new_cert_id + "/RC" + + return new_cert_id + + def _canonical_in(self): + return self.clean.replace(" ", "") + + def _canonical_se(self): + return self.clean.replace(" ", "") + + def _canonical_uk(self): + new_cert_id = self.clean + if match := re.match("CERTIFICATION REPORT No. P([0-9]+[A-Z]?)", new_cert_id): + new_cert_id = "CRP" + match.group(1) + return new_cert_id + + def _canonical_ca(self): + new_cert_id = self.clean + if new_cert_id.endswith("-CR"): + new_cert_id = new_cert_id[:-3] + if new_cert_id.endswith("P"): + new_cert_id = new_cert_id[:-1] + return new_cert_id.replace(" ", "-") + + def _canonical_jp(self): + new_cert_id = self.clean + if match := re.match("Certification No. (C[0-9]+)", new_cert_id): + return match.group(1) + if match := re.search("CRP-(C[0-9]+)-", new_cert_id): + return match.group(1) + return new_cert_id + + def _canonical_no(self): + new_cert_id = self.clean + cert_num = int(new_cert_id.split("-")[1]) + return f"SERTIT-{cert_num:03}" + + @property + def clean(self) -> str: + """ + The clean version of this certificate id. + """ + return self.raw.replace("\N{HYPHEN}", "-").strip() + + @property + def canonical(self) -> str: + """ + The canonical version of this certificate id. + """ + # We have rules for some schemes to make canonical cert_ids. + schemes = { + "FR": self._canonical_fr, + "DE": self._canonical_de, + "ES": self._canonical_es, + "IT": self._canonical_it, + "IN": self._canonical_in, + "SE": self._canonical_se, + "UK": self._canonical_uk, + "CA": self._canonical_ca, + "JP": self._canonical_jp, + "NO": self._canonical_no, + } + + if self.scheme in schemes: + return schemes[self.scheme]() + else: + return self.clean + + +def canonicalize(cert_id_str: str, scheme: str) -> str: + return CertificateId(scheme, cert_id_str).canonical diff --git a/src/sec_certs/sample/cc_maintenance_update.py b/src/sec_certs/sample/cc_maintenance_update.py new file mode 100644 index 00000000..abe3b176 --- /dev/null +++ b/src/sec_certs/sample/cc_maintenance_update.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import logging +from datetime import date +from typing import ClassVar + +import sec_certs.utils.helpers as helpers +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.serialization.json import ComplexSerializableType + +logger = logging.getLogger(__name__) + + +class CommonCriteriaMaintenanceUpdate(CommonCriteriaCert, ComplexSerializableType): + pandas_columns: ClassVar[list[str]] = [ + "dgst", + "name", + "report_link", + "st_link", + "related_cert_digest", + "maintenance_date", + ] + + def __init__( + self, + name: str, + report_link: str, + st_link: str, + state: CommonCriteriaCert.InternalState | None, + pdf_data: CommonCriteriaCert.PdfData | None, + heuristics: CommonCriteriaCert.Heuristics | None, + related_cert_digest: str, + maintenance_date: date, + ): + super().__init__( + "", + "", + name, + "", + "", + "", + None, + None, + report_link, + st_link, + "", + "", + set(), + set(), + state, + pdf_data, + heuristics, + ) + self.related_cert_digest = related_cert_digest + self.maintenance_date = maintenance_date + + @property + def serialized_attributes(self) -> list[str]: + return ["dgst"] + list(self.__class__.__init__.__code__.co_varnames)[1:] + + @property + def dgst(self) -> str: + if not self.name: + raise RuntimeError("MaintenanceUpdate digest can't be computed, because name of update is missing.") + return "cert_" + self.related_cert_digest + "_update_" + helpers.get_first_16_bytes_sha256(self.name) + + @property + def pandas_tuple(self) -> tuple: + return tuple([getattr(self, x) for x in CommonCriteriaMaintenanceUpdate.pandas_columns]) + + @classmethod + def from_dict(cls, dct: dict) -> CommonCriteriaMaintenanceUpdate: + dct.pop("dgst") + return cls(*(tuple(dct.values()))) + + @classmethod + def get_updates_from_cc_cert(cls, cert: CommonCriteriaCert) -> list[CommonCriteriaMaintenanceUpdate]: + if cert.maintenance_updates is None: + raise RuntimeError("Dataset was probably not built correctly - this should not be happening.") + + return [ + cls( + x.maintenance_title, + x.maintenance_report_link, + x.maintenance_st_link, + None, + None, + None, + cert.dgst, + x.maintenance_date, + ) + for x in cert.maintenance_updates + if ( + x.maintenance_title is not None + and x.maintenance_report_link is not None + and x.maintenance_st_link is not None + and x.maintenance_date is not None + ) + ] diff --git a/src/sec_certs/sample/certificate.py b/src/sec_certs/sample/certificate.py new file mode 100644 index 00000000..bb49c0df --- /dev/null +++ b/src/sec_certs/sample/certificate.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import copy +import logging +from abc import ABC, abstractmethod +from collections import ChainMap +from dataclasses import dataclass, field +from typing import Any, Generic, TypeVar + +import sec_certs.utils.extract +from sec_certs.cert_rules import PANDAS_KEYWORDS_CATEGORIES +from sec_certs.serialization.json import ComplexSerializableType + +logger = logging.getLogger(__name__) + +T = TypeVar("T", bound="Certificate") +H = TypeVar("H", bound="Heuristics") +P = TypeVar("P", bound="PdfData") + + +@dataclass +class References(ComplexSerializableType): + directly_referenced_by: set[str] | None = field(default=None) + indirectly_referenced_by: set[str] | None = field(default=None) + directly_referencing: set[str] | None = field(default=None) + indirectly_referencing: set[str] | None = field(default=None) + + +class Heuristics: + cpe_matches: set[str] | None + related_cves: set[str] | None + + +class PdfData: + def get_keywords_df_data(self, var: str) -> dict[str, float]: + data_dct = getattr(self, var) + return dict( + ChainMap( + *[ + sec_certs.utils.extract.get_sums_for_rules_subset(data_dct, cat) + for cat in PANDAS_KEYWORDS_CATEGORIES + ] + ) + ) + + +class Certificate(Generic[T, H, P], ABC, ComplexSerializableType): + manufacturer: str | None + name: str | None + pdf_data: P + heuristics: H + + def __init__(self, *args, **kwargs): + pass + + def __repr__(self) -> str: + return str(self.to_dict()) + + def __str__(self) -> str: + return "Not implemented" + + @property + @abstractmethod + def dgst(self): + raise NotImplementedError("Not meant to be implemented") + + @property + @abstractmethod + def label_studio_title(self): + raise NotImplementedError("Not meant to be implemented") + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Certificate): + return False + return self.dgst == other.dgst + + def to_dict(self) -> dict[str, Any]: + return { + **{"dgst": self.dgst}, + **{key: val for key, val in copy.deepcopy(self.__dict__).items() if key in self.serialized_attributes}, + } + + @classmethod + def from_dict(cls: type[T], dct: dict) -> T: + dct.pop("dgst") + return cls(**dct) + + @abstractmethod + def compute_heuristics_version(self) -> None: + raise NotImplementedError("Not meant to be implemented") diff --git a/src/sec_certs/sample/common_criteria.py b/src/sec_certs/sample/common_criteria.py new file mode 100644 index 00000000..acd54178 --- /dev/null +++ b/src/sec_certs/sample/common_criteria.py @@ -0,0 +1,986 @@ +from __future__ import annotations + +import copy +import re +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import date, datetime +from enum import Enum +from pathlib import Path +from typing import Any, ClassVar +from urllib.parse import unquote_plus, urlparse + +import numpy as np +import requests +from bs4 import Tag + +import sec_certs.utils.extract +import sec_certs.utils.pdf +import sec_certs.utils.sanitization +from sec_certs import constants as constants +from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, cc_rules, rules, security_level_csv_scan +from sec_certs.sample.cc_certificate_id import canonicalize +from sec_certs.sample.certificate import Certificate +from sec_certs.sample.certificate import Heuristics as BaseHeuristics +from sec_certs.sample.certificate import PdfData as BasePdfData +from sec_certs.sample.certificate import References, logger +from sec_certs.sample.protection_profile import ProtectionProfile +from sec_certs.sample.sar import SAR +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.serialization.pandas import PandasSerializableType +from sec_certs.utils import helpers +from sec_certs.utils.extract import normalize_match_string + +HEADERS = { + "anssi": sec_certs.utils.extract.search_only_headers_anssi, + "bsi": sec_certs.utils.extract.search_only_headers_bsi, + "nscib": sec_certs.utils.extract.search_only_headers_nscib, + "niap": sec_certs.utils.extract.search_only_headers_niap, + "canada": sec_certs.utils.extract.search_only_headers_canada, +} + + +class ReferenceType(Enum): + DIRECT = "direct" + INDIRECT = "indirect" + + +class CommonCriteriaCert( + Certificate["CommonCriteriaCert", "CommonCriteriaCert.Heuristics", "CommonCriteriaCert.PdfData"], + PandasSerializableType, + ComplexSerializableType, +): + """ + Data structure for common criteria certificate. Contains several inner classes that layer the data logic. + Can be serialized into/from json (`ComplexSerializableType`) or pandas (`PandasSerializableType)`. + Is basic element of `CCDataset`. The functionality is mostly related to holding data and transformations that + the certificate can handle itself. `CCDataset` class then instrument this functionality. + """ + + cc_url = "http://www.commoncriteriaportal.org" + empty_st_url = "http://www.commoncriteriaportal.org/files/epfiles/" + + @dataclass(eq=True, frozen=True) + class MaintenanceReport(ComplexSerializableType): + """ + Object for holding maintenance reports. + """ + + maintenance_date: date | None + maintenance_title: str | None + maintenance_report_link: str | None + maintenance_st_link: str | None + + def __post_init__(self): + super().__setattr__( + "maintenance_report_link", sec_certs.utils.sanitization.sanitize_link(self.maintenance_report_link) + ) + super().__setattr__( + "maintenance_st_link", sec_certs.utils.sanitization.sanitize_link(self.maintenance_st_link) + ) + super().__setattr__( + "maintenance_title", sec_certs.utils.sanitization.sanitize_string(self.maintenance_title) + ) + super().__setattr__("maintenance_date", sec_certs.utils.sanitization.sanitize_date(self.maintenance_date)) + + @classmethod + def from_dict(cls, dct: dict) -> CommonCriteriaCert.MaintenanceReport: + new_dct = dct.copy() + new_dct["maintenance_date"] = ( + date.fromisoformat(dct["maintenance_date"]) + if isinstance(dct["maintenance_date"], str) + else dct["maintenance_date"] + ) + return super().from_dict(new_dct) + + def __lt__(self, other): + return self.maintenance_date < other.maintenance_date + + @dataclass(init=False) + class InternalState(ComplexSerializableType): + """ + Holds internal state of the certificate, whether downloads and converts of individual components succeeded. Also + holds information about errors and paths to the files. + """ + + st_download_ok: bool # Whether target download went OK + report_download_ok: bool # Whether report download went OK + st_convert_garbage: bool # Whether initial target conversion resulted in garbage + report_convert_garbage: bool # Whether initial report conversion resulted in garbage + st_convert_ok: bool # Whether overall target conversion went OK (either pdftotext or via OCR) + report_convert_ok: bool # Whether overall report conversion went OK (either pdftotext or via OCR) + st_extract_ok: bool # Whether target extraction went OK + report_extract_ok: bool # Whether report extraction went OK + + st_pdf_hash: str | None + report_pdf_hash: str | None + st_txt_hash: str | None + report_txt_hash: str | None + + st_pdf_path: Path + report_pdf_path: Path + st_txt_path: Path + report_txt_path: Path + + def __init__( + self, + st_download_ok: bool = False, + report_download_ok: bool = False, + st_convert_garbage: bool = False, + report_convert_garbage: bool = False, + st_convert_ok: bool = False, + report_convert_ok: bool = False, + st_extract_ok: bool = False, + report_extract_ok: bool = False, + st_pdf_hash: str | None = None, + report_pdf_hash: str | None = None, + st_txt_hash: str | None = None, + report_txt_hash: str | None = None, + ): + super().__init__() + self.st_download_ok = st_download_ok + self.report_download_ok = report_download_ok + self.st_convert_garbage = st_convert_garbage + self.report_convert_garbage = report_convert_garbage + self.st_convert_ok = st_convert_ok + self.report_convert_ok = report_convert_ok + self.st_extract_ok = st_extract_ok + self.report_extract_ok = report_extract_ok + self.st_pdf_hash = st_pdf_hash + self.report_pdf_hash = report_pdf_hash + self.st_txt_hash = st_txt_hash + self.report_txt_hash = report_txt_hash + + @property + def serialized_attributes(self) -> list[str]: + return [ + "st_download_ok", + "report_download_ok", + "st_convert_garbage", + "report_convert_garbage", + "st_convert_ok", + "report_convert_ok", + "st_extract_ok", + "report_extract_ok", + "st_pdf_hash", + "report_pdf_hash", + "st_txt_hash", + "report_txt_hash", + ] + + def report_is_ok_to_download(self, fresh: bool = True) -> bool: + return True if fresh else not self.report_download_ok + + def st_is_ok_to_download(self, fresh: bool = True) -> bool: + return True if fresh else not self.st_download_ok + + def report_is_ok_to_convert(self, fresh: bool = True) -> bool: + return self.report_download_ok if fresh else self.report_download_ok and not self.report_convert_ok + + def st_is_ok_to_convert(self, fresh: bool = True) -> bool: + return self.st_download_ok if fresh else self.st_download_ok and not self.st_convert_ok + + def report_is_ok_to_analyze(self, fresh: bool = True) -> bool: + if fresh is True: + return self.report_download_ok and self.report_convert_ok + else: + return self.report_download_ok and self.report_convert_ok and not self.report_extract_ok + + def st_is_ok_to_analyze(self, fresh: bool = True) -> bool: + if fresh is True: + return self.st_download_ok and self.st_convert_ok + else: + return self.st_download_ok and self.st_convert_ok and not self.st_extract_ok + + @dataclass + class PdfData(BasePdfData, ComplexSerializableType): + """ + Class that holds data extracted from pdf files. + """ + + report_metadata: dict[str, Any] | None = field(default=None) + st_metadata: dict[str, Any] | None = field(default=None) + report_frontpage: dict[str, dict[str, Any]] | None = field(default=None) + st_frontpage: dict[str, dict[str, Any]] | None = field(default=None) + report_keywords: dict[str, Any] | None = field(default=None) + st_keywords: dict[str, Any] | None = field(default=None) + report_filename: str | None = field(default=None) + st_filename: str | None = field(default=None) + + def __bool__(self) -> bool: + return any([x is not None for x in vars(self)]) + + @property + def bsi_data(self) -> dict[str, Any] | None: + """ + Returns frontpage data related to BSI-provided information + """ + return self.report_frontpage.get("bsi", None) if self.report_frontpage else None + + @property + def niap_data(self) -> dict[str, Any] | None: + """ + Returns frontpage data related to niap-provided information + """ + return self.report_frontpage.get("niap", None) if self.report_frontpage else None + + @property + def nscib_data(self) -> dict[str, Any] | None: + """ + Returns frontpage data related to nscib-provided information + """ + return self.report_frontpage.get("nscib", None) if self.report_frontpage else None + + @property + def canada_data(self) -> dict[str, Any] | None: + """ + Returns frontpage data related to canada-provided information + """ + return self.report_frontpage.get("canada", None) if self.report_frontpage else None + + @property + def anssi_data(self) -> dict[str, Any] | None: + """ + Returns frontpage data related to ANSSI-provided information + """ + return self.report_frontpage.get("anssi", None) if self.report_frontpage else None + + @property + def cert_lab(self) -> list[str] | None: + """ + Returns labs for which certificate data was parsed. + """ + labs = [ + data["cert_lab"].split(" ")[0].upper() + for data in [self.bsi_data, self.anssi_data, self.niap_data, self.nscib_data, self.canada_data] + if data + ] + return labs if labs else None + + @property + def bsi_cert_id(self) -> str | None: + return self.bsi_data.get("cert_id", None) if self.bsi_data else None + + @property + def niap_cert_id(self) -> str | None: + return self.niap_data.get("cert_id", None) if self.niap_data else None + + @property + def nscib_cert_id(self) -> str | None: + return self.nscib_data.get("cert_id", None) if self.nscib_data else None + + @property + def canada_cert_id(self) -> str | None: + return self.canada_data.get("cert_id", None) if self.canada_data else None + + @property + def anssi_cert_id(self) -> str | None: + return self.anssi_data.get("cert_id", None) if self.anssi_data else None + + def frontpage_cert_id(self, scheme: str) -> dict[str, float]: + """ + Get cert_id candidate from the frontpage of the report. + """ + scheme_map = { + "DE": self.bsi_cert_id, + "US": self.niap_cert_id, + "NL": self.nscib_cert_id, + "CA": self.canada_cert_id, + "FR": self.anssi_cert_id, + } + if scheme in scheme_map and (candidate := scheme_map[scheme]): + return {candidate: 1.0} + return {} + + def filename_cert_id(self, scheme: str) -> dict[str, float]: + """ + Get cert_id candidates from the matches in the report filename. + """ + if not self.report_filename: + return {} + scheme_rules = rules["cc_cert_id"][scheme] + matches: Counter = Counter() + for rule in scheme_rules: + match = re.search(rule, self.report_filename) + if match: + cert_id = normalize_match_string(match.group()) + matches[cert_id] += 1 + if not matches: + return {} + total = max(matches.values()) + results = {} + for candidate, count in matches.items(): + results[candidate] = count / total + # TODO count length in weight + return results + + def keywords_cert_id(self, scheme: str) -> dict[str, float]: + """ + Get cert_id candidates from the keywords matches in the report. + """ + if not self.report_keywords: + return {} + cert_id_matches = self.report_keywords.get("cc_cert_id") + if not cert_id_matches: + return {} + + if scheme not in cert_id_matches: + return {} + matches: Counter = Counter(cert_id_matches[scheme]) + if not matches: + return {} + total = max(matches.values()) + results = {} + for candidate, count in matches.items(): + results[candidate] = count / total + # TODO count length in weight + return results + + def metadata_cert_id(self, scheme: str) -> dict[str, float]: + """ + Get cert_id candidates from the report metadata. + """ + scheme_rules = rules["cc_cert_id"][scheme] + fields = ("/Title", "/Subject") + matches: Counter = Counter() + for meta_field in fields: + field_val = self.report_metadata.get(meta_field) if self.report_metadata else None + if not field_val: + continue + for rule in scheme_rules: + match = re.search(rule, field_val) + if match: + cert_id = normalize_match_string(match.group()) + matches[cert_id] += 1 + if not matches: + return {} + total = max(matches.values()) + results = {} + for candidate, count in matches.items(): + results[candidate] = count / total + # TODO count length in weight + return results + + def candidate_cert_ids(self, scheme: str) -> dict[str, float]: + frontpage_id = self.frontpage_cert_id(scheme) + metadata_id = self.metadata_cert_id(scheme) + filename_id = self.filename_cert_id(scheme) + keywords_id = self.keywords_cert_id(scheme) + + # Join them and weigh them, each is normalized with weights from 0 to 1 (if anything is returned) + candidates: dict[str, float] = defaultdict(lambda: 0.0) + # TODO: Add heuristic based on ordering of ids (and extracted year + increment) + # TODO: Add heuristic based on length + for candidate, count in frontpage_id.items(): + candidates[canonicalize(candidate, scheme)] += count * 1.5 + for candidate, count in metadata_id.items(): + candidates[canonicalize(candidate, scheme)] += count * 1.2 + for candidate, count in keywords_id.items(): + candidates[canonicalize(candidate, scheme)] += count * 1.0 + for candidate, count in filename_id.items(): + candidates[canonicalize(candidate, scheme)] += count * 1.0 + return candidates + + @dataclass + class Heuristics(BaseHeuristics, ComplexSerializableType): + """ + Class for various heuristics related to CommonCriteriaCert + """ + + extracted_versions: set[str] | None = field(default=None) + cpe_matches: set[str] | None = field(default=None) + verified_cpe_matches: set[str] | None = field(default=None) + related_cves: set[str] | None = field(default=None) + cert_lab: list[str] | None = field(default=None) + cert_id: str | None = field(default=None) + st_references: References = field(default_factory=References) + report_references: References = field(default_factory=References) + extracted_sars: set[SAR] | None = field(default=None) + direct_transitive_cves: set[str] | None = field(default=None) + indirect_transitive_cves: set[str] | None = field(default=None) + + @property + def serialized_attributes(self) -> list[str]: + return copy.deepcopy(super().serialized_attributes) + + pandas_columns: ClassVar[list[str]] = [ + "dgst", + "cert_id", + "name", + "status", + "category", + "manufacturer", + "scheme", + "security_level", + "eal", + "not_valid_before", + "not_valid_after", + "report_link", + "st_link", + "cert_link", + "manufacturer_web", + "extracted_versions", + "cpe_matches", + "verified_cpe_matches", + "related_cves", + "directly_referenced_by", + "indirectly_referenced_by", + "directly_referencing", + "indirectly_referencing", + "extracted_sars", + "protection_profiles", + "cert_lab", + ] + + def __init__( + self, + status: str, + category: str, + name: str, + manufacturer: str | None, + scheme: str, + security_level: str | set[str], + not_valid_before: date | None, + not_valid_after: date | None, + report_link: str, + st_link: str, + cert_link: str | None, + manufacturer_web: str | None, + protection_profiles: set[ProtectionProfile] | None, + maintenance_updates: set[MaintenanceReport] | None, + state: InternalState | None, + pdf_data: PdfData | None, + heuristics: Heuristics | None, + ): + super().__init__() + + self.status = status + self.category = category + self.name = sec_certs.utils.sanitization.sanitize_string(name) + + self.manufacturer = None + if manufacturer: + self.manufacturer = sec_certs.utils.sanitization.sanitize_string(manufacturer) + + self.scheme = scheme + self.security_level = sec_certs.utils.sanitization.sanitize_security_levels(security_level) + self.not_valid_before = sec_certs.utils.sanitization.sanitize_date(not_valid_before) + self.not_valid_after = sec_certs.utils.sanitization.sanitize_date(not_valid_after) + self.report_link = sec_certs.utils.sanitization.sanitize_link(report_link) + self.st_link = sec_certs.utils.sanitization.sanitize_link(st_link) + self.cert_link = sec_certs.utils.sanitization.sanitize_link(cert_link) + self.manufacturer_web = sec_certs.utils.sanitization.sanitize_link(manufacturer_web) + self.protection_profiles = protection_profiles + self.maintenance_updates = maintenance_updates + self.state = self.InternalState() if not state else state + self.pdf_data = self.PdfData() if not pdf_data else pdf_data + self.heuristics: CommonCriteriaCert.Heuristics = self.Heuristics() if not heuristics else heuristics + + @property + def dgst(self) -> str: + """ + Computes the primary key of the sample using first 16 bytes of SHA-256 digest + """ + if not (self.name is not None and self.report_link is not None and self.category is not None): + raise RuntimeError("Certificate digest can't be computed, because information is missing.") + return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link) + + @property + def eal(self) -> str | None: + """ + Returns EAL of certificate if it was extracted, None otherwise. + """ + res = [x for x in self.security_level if re.match(security_level_csv_scan, x)] + if res and len(res) == 1: + return res[0] + if res and len(res) > 1: + raise ValueError(f"Expected single EAL in security_level field, got: {res}") + else: + if self.protection_profiles: + return helpers.choose_lowest_eal({x.pp_eal for x in self.protection_profiles if x.pp_eal}) + else: + return None + + @property + def actual_sars(self) -> set[SAR] | None: + """ + Computes actual SARs. First, SARs implied by EAL are computed. Then, these are augmented with heuristically extracted SARs + :return Optional[Set[SAR]]: Set of actual SARs of a certificate, None if empty + """ + sars = dict() + if self.eal: + sars = {x[0]: SAR(x[0], x[1]) for x in SARS_IMPLIED_FROM_EAL[self.eal[:4]]} + + if self.heuristics.extracted_sars: + for sar in self.heuristics.extracted_sars: + if sar not in sars or sar.level > sars[sar.family].level: + sars[sar.family] = sar + + return set(sars.values()) if sars else None + + @property + def label_studio_title(self) -> str | None: + return self.name + + @property + def pandas_tuple(self) -> tuple: + """ + Returns tuple of attributes meant for pandas serialization + """ + return ( + self.dgst, + self.heuristics.cert_id, + self.name, + self.status, + self.category, + self.manufacturer, + self.scheme, + self.security_level, + self.eal, + self.not_valid_before, + self.not_valid_after, + self.report_link, + self.st_link, + self.cert_link, + self.manufacturer_web, + self.heuristics.extracted_versions, + self.heuristics.cpe_matches, + self.heuristics.verified_cpe_matches, + self.heuristics.related_cves, + self.heuristics.report_references.directly_referenced_by, + self.heuristics.report_references.indirectly_referenced_by, + self.heuristics.report_references.directly_referencing, + self.heuristics.report_references.indirectly_referencing, + self.heuristics.extracted_sars, + [x.pp_name for x in self.protection_profiles] if self.protection_profiles else np.nan, + self.heuristics.cert_lab[0] if (self.heuristics.cert_lab and self.heuristics.cert_lab[0]) else np.nan, + ) + + def __str__(self) -> str: + printed_manufacturer = self.manufacturer if self.manufacturer else "Unknown manufacturer" + return str(printed_manufacturer) + " " + str(self.name) + " dgst: " + self.dgst + + def merge(self, other: CommonCriteriaCert, other_source: str | None = None) -> None: + """ + Merges with other CC sample. Assuming they come from different sources, e.g., csv and html. + Assuming that html source has better protection profiles, they overwrite CSV info + On other values the sanity checks are made. + """ + if self != other: + logger.warning( + f"Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}" + ) + + for att, val in vars(self).items(): + if not val: + setattr(self, att, getattr(other, att)) + elif other_source == "html" and att == "protection_profiles": + setattr(self, att, getattr(other, att)) + elif other_source == "html" and att == "maintenance_updates": + setattr(self, att, getattr(other, att)) + elif att == "state": + setattr(self, att, getattr(other, att)) + else: + if getattr(self, att) != getattr(other, att): + logger.warning( + f"When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}" + ) + + @classmethod + def from_dict(cls, dct: dict) -> CommonCriteriaCert: + """ + Deserializes dictionary into `CommonCriteriaCert` + """ + new_dct = dct.copy() + new_dct["maintenance_updates"] = set(dct["maintenance_updates"]) + new_dct["protection_profiles"] = set(dct["protection_profiles"]) + new_dct["not_valid_before"] = ( + date.fromisoformat(dct["not_valid_before"]) + if isinstance(dct["not_valid_before"], str) + else dct["not_valid_before"] + ) + new_dct["not_valid_after"] = ( + date.fromisoformat(dct["not_valid_after"]) + if isinstance(dct["not_valid_after"], str) + else dct["not_valid_after"] + ) + return super(cls, CommonCriteriaCert).from_dict(new_dct) + + @staticmethod + def _html_row_get_name(cell: Tag) -> str: + return list(cell.stripped_strings)[0] + + @staticmethod + def _html_row_get_manufacturer(cell: Tag) -> str | None: + if lst := list(cell.stripped_strings): + return lst[0] + else: + return None + + @staticmethod + def _html_row_get_scheme(cell: Tag) -> str: + return list(cell.stripped_strings)[0] + + @staticmethod + def _html_row_get_security_level(cell: Tag) -> set: + return set(cell.stripped_strings) + + @staticmethod + def _html_row_get_manufacturer_web(cell: Tag) -> str | None: + for link in cell.find_all("a"): + if link is not None and link.get("title") == "Vendor's web site" and link.get("href") != "http://": + return link.get("href") + return None + + @staticmethod + def _html_row_get_protection_profiles(cell: Tag) -> set: + protection_profiles = set() + for link in list(cell.find_all("a")): + if link.get("href") is not None and "/ppfiles/" in link.get("href"): + protection_profiles.add( + ProtectionProfile( + pp_name=str(link.contents[0]), pp_eal=None, pp_link=CommonCriteriaCert.cc_url + link.get("href") + ) + ) + return protection_profiles + + @staticmethod + def _html_row_get_date(cell: Tag) -> date | None: + text = cell.get_text() + extracted_date = datetime.strptime(text, "%Y-%m-%d").date() if text else None + return extracted_date + + @staticmethod + def _html_row_get_report_st_links(cell: Tag) -> tuple[str, str]: + links = cell.find_all("a") + assert links[1].get("title").startswith("Certification Report") + assert links[2].get("title").startswith("Security Target") + + report_link = CommonCriteriaCert.cc_url + links[1].get("href") + security_target_link = CommonCriteriaCert.cc_url + links[2].get("href") + + return report_link, security_target_link + + @staticmethod + def _html_row_get_cert_link(cell: Tag) -> str | None: + links = cell.find_all("a") + return CommonCriteriaCert.cc_url + links[0].get("href") if links else None + + @staticmethod + def _html_row_get_maintenance_div(cell: Tag) -> Tag | None: + divs = cell.find_all("div") + for d in divs: + if d.find("div") and d.stripped_strings and list(d.stripped_strings)[0] == "Maintenance Report(s)": + return d + return None + + @staticmethod + def _html_row_get_maintenance_updates(main_div: Tag) -> set[CommonCriteriaCert.MaintenanceReport]: + possible_updates = list(main_div.find_all("li")) + maintenance_updates = set() + for u in possible_updates: + text = list(u.stripped_strings)[0] + main_date = datetime.strptime(text.split(" ")[0], "%Y-%m-%d").date() if text else None + main_title = text.split("– ")[1] + main_report_link = None + main_st_link = None + links = u.find_all("a") + for link in links: + if link.get("title").startswith("Maintenance Report:"): + main_report_link = CommonCriteriaCert.cc_url + link.get("href") + elif link.get("title").startswith("Maintenance ST"): + main_st_link = CommonCriteriaCert.cc_url + link.get("href") + else: + logger.error("Unknown link in Maintenance part!") + maintenance_updates.add( + CommonCriteriaCert.MaintenanceReport(main_date, main_title, main_report_link, main_st_link) + ) + return maintenance_updates + + @classmethod + def from_html_row(cls, row: Tag, status: str, category: str) -> CommonCriteriaCert: + """ + Creates a CC sample from html row of commoncriteria.org webpage. + """ + + cells = list(row.find_all("td")) + if len(cells) != 7: + raise ValueError(f"Unexpected number of
elements in CC html row. Expected: 7, actual: {len(cells)}") + + name = CommonCriteriaCert._html_row_get_name(cells[0]) + manufacturer = CommonCriteriaCert._html_row_get_manufacturer(cells[1]) + manufacturer_web = CommonCriteriaCert._html_row_get_manufacturer_web(cells[1]) + scheme = CommonCriteriaCert._html_row_get_scheme(cells[6]) + security_level = CommonCriteriaCert._html_row_get_security_level(cells[5]) + protection_profiles = CommonCriteriaCert._html_row_get_protection_profiles(cells[0]) + not_valid_before = CommonCriteriaCert._html_row_get_date(cells[3]) + not_valid_after = CommonCriteriaCert._html_row_get_date(cells[4]) + report_link, st_link = CommonCriteriaCert._html_row_get_report_st_links(cells[0]) + cert_link = CommonCriteriaCert._html_row_get_cert_link(cells[2]) + maintenance_div = CommonCriteriaCert._html_row_get_maintenance_div(cells[0]) + maintenances = ( + CommonCriteriaCert._html_row_get_maintenance_updates(maintenance_div) if maintenance_div else set() + ) + + return cls( + status, + category, + name, + manufacturer, + scheme, + security_level, + not_valid_before, + not_valid_after, + report_link, + st_link, + cert_link, + manufacturer_web, + protection_profiles, + maintenances, + None, + None, + None, + ) + + def set_local_paths( + self, + report_pdf_dir: str | Path | None, + st_pdf_dir: str | Path | None, + report_txt_dir: str | Path | None, + st_txt_dir: str | Path | None, + ) -> None: + """ + Sets paths to files given the requested directories + + :param Optional[Union[str, Path]] report_pdf_dir: Directory where pdf reports shall be stored + :param Optional[Union[str, Path]] st_pdf_dir: Directory where pdf security targets shall be stored + :param Optional[Union[str, Path]] report_txt_dir: Directory where txt reports shall be stored + :param Optional[Union[str, Path]] st_txt_dir: Directory where txt security targets shall be stored + """ + if report_pdf_dir is not None: + self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf") + if st_pdf_dir is not None: + self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf") + if report_txt_dir is not None: + self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + ".txt") + if st_txt_dir is not None: + self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + ".txt") + + @staticmethod + def download_pdf_report(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Downloads pdf of certification report given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to download the pdf report for + :return CommonCriteriaCert: returns the modified certificate with updated state + """ + exit_code: str | int + if not cert.report_link: + exit_code = "No link" + else: + exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path) + if exit_code != requests.codes.ok: + error_msg = f"failed to download report from {cert.report_link}, code: {exit_code}" + logger.error(f"Cert dgst: {cert.dgst} " + error_msg) + cert.state.report_download_ok = False + else: + cert.state.report_download_ok = True + cert.state.report_pdf_hash = helpers.get_sha256_filepath(cert.state.report_pdf_path) + cert.pdf_data.report_filename = unquote_plus(str(urlparse(cert.report_link).path).split("/")[-1]) + return cert + + @staticmethod + def download_pdf_st(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Downloads pdf of security target given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to download the pdf security target for + :return CommonCriteriaCert: returns the modified certificate with updated state + """ + exit_code: str | int + if not cert.st_link: + exit_code = "No link" + else: + exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path) + if exit_code != requests.codes.ok: + error_msg = f"failed to download ST from {cert.st_link}, code: {exit_code}" + logger.error(f"Cert dgst: {cert.dgst} " + error_msg) + cert.state.st_download_ok = False + else: + cert.state.st_download_ok = True + cert.state.st_pdf_hash = helpers.get_sha256_filepath(cert.state.st_pdf_path) + cert.pdf_data.st_filename = unquote_plus(str(urlparse(cert.st_link).path).split("/")[-1]) + return cert + + @staticmethod + def convert_report_pdf(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Converts the pdf certification report to txt, given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to download the pdf report for + :return CommonCriteriaCert: the modified certificate with updated state + """ + ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file( + cert.state.report_pdf_path, cert.state.report_txt_path + ) + # If OCR was done the result was garbage + cert.state.report_convert_garbage = ocr_done + # And put the whole result into convert_ok + cert.state.report_convert_ok = ok_result + if not ok_result: + error_msg = "failed to convert report pdf->txt" + logger.error(f"Cert dgst: {cert.dgst} " + error_msg) + else: + cert.state.report_txt_hash = helpers.get_sha256_filepath(cert.state.report_txt_path) + return cert + + @staticmethod + def convert_st_pdf(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Converts the pdf security target to txt, given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to download the pdf security target for + :return CommonCriteriaCert: the modified certificate with updated state + """ + ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path) + # If OCR was done the result was garbage + cert.state.st_convert_garbage = ocr_done + # And put the whole result into convert_ok + cert.state.st_convert_ok = ok_result + if not ok_result: + error_msg = "failed to convert security target pdf->txt" + logger.error(f"Cert dgst: {cert.dgst} " + error_msg) + else: + cert.state.st_txt_hash = helpers.get_sha256_filepath(cert.state.st_txt_path) + return cert + + @staticmethod + def extract_st_pdf_metadata(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Extracts metadata from security target pdf given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to extract the metadata for. + :return CommonCriteriaCert: the modified certificate with updated state + """ + response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path) + if response != constants.RETURNCODE_OK: + cert.state.st_extract_ok = False + else: + cert.state.st_extract_ok = True + return cert + + @staticmethod + def extract_report_pdf_metadata(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Extracts metadata from certification report pdf given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to extract the metadata for. + :return CommonCriteriaCert: the modified certificate with updated state + """ + response, cert.pdf_data.report_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.report_pdf_path) + if response != constants.RETURNCODE_OK: + cert.state.report_extract_ok = False + else: + cert.state.report_extract_ok = True + return cert + + @staticmethod + def extract_st_pdf_frontpage(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Extracts data from security target pdf frontpage given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to extract the frontpage data for. + :return CommonCriteriaCert: the modified certificate with updated state + """ + cert.pdf_data.st_frontpage = {} + + for header_type, associated_header_func in HEADERS.items(): + response, cert.pdf_data.st_frontpage[header_type] = associated_header_func(cert.state.st_txt_path) + + if response != constants.RETURNCODE_OK: + cert.state.st_extract_ok = False + return cert + + @staticmethod + def extract_report_pdf_frontpage(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Extracts data from certification report pdf frontpage given the certificate. Staticmethod to allow for parallelization. + + :param CommonCriteriaCert cert: cert to extract the frontpage data for. + :return CommonCriteriaCert: the modified certificate with updated state + """ + cert.pdf_data.report_frontpage = {} + + for header_type, associated_header_func in HEADERS.items(): + response, cert.pdf_data.report_frontpage[header_type] = associated_header_func(cert.state.report_txt_path) + + if response != constants.RETURNCODE_OK: + cert.state.report_extract_ok = False + return cert + + @staticmethod + def extract_report_pdf_keywords(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Matches regular expresions in txt obtained from certification report and extracts the matches into attribute. + Static method to allow for parallelization + + :param CommonCriteriaCert cert: certificate to extract the keywords for. + :return CommonCriteriaCert: the modified certificate with extracted keywords. + """ + report_keywords = sec_certs.utils.extract.extract_keywords(cert.state.report_txt_path, cc_rules) + if report_keywords is None: + cert.state.report_extract_ok = False + else: + cert.pdf_data.report_keywords = report_keywords + return cert + + @staticmethod + def extract_st_pdf_keywords(cert: CommonCriteriaCert) -> CommonCriteriaCert: + """ + Matches regular expresions in txt obtained from security target and extracts the matches into attribute. + Static method to allow for parallelization + + :param CommonCriteriaCert cert: certificate to extract the keywords for. + :return CommonCriteriaCert: the modified certificate with extracted keywords. + """ + st_keywords = sec_certs.utils.extract.extract_keywords(cert.state.st_txt_path, cc_rules) + if st_keywords is None: + cert.state.st_extract_ok = False + else: + cert.pdf_data.st_keywords = st_keywords + return cert + + def compute_heuristics_version(self) -> None: + """ + Fills in the heuristically obtained version of certified product into attribute in heuristics class. + """ + self.heuristics.extracted_versions = helpers.compute_heuristics_version(self.name) if self.name else set() + + def compute_heuristics_cert_lab(self) -> None: + """ + Fills in the heuristically obtained evaluation laboratory into attribute in heuristics class. + """ + if not self.pdf_data: + logger.error("Cannot compute sample lab when pdf files were not processed.") + return + self.heuristics.cert_lab = self.pdf_data.cert_lab + + def compute_heuristics_cert_id(self): + """ + Compute the heuristics cert_id of this cert, using several methods. + + The candidate cert_ids are extracted from the frontpage, PDF metadata, filename, and keywords matches. + + Finally, the cert_id is canonicalized. + """ + if not self.pdf_data: + logger.warning("Cannot compute sample id when pdf files were not processed.") + return + # Extract candidate cert_ids + candidates = self.pdf_data.candidate_cert_ids(self.scheme) + + if candidates: + max_weight = max(candidates.values()) + max_candidates = list(filter(lambda x: candidates[x] == max_weight, candidates.keys())) + max_candidates.sort(key=len, reverse=True) + self.heuristics.cert_id = max_candidates[0] diff --git a/src/sec_certs/sample/cpe.py b/src/sec_certs/sample/cpe.py new file mode 100644 index 00000000..a7532f7f --- /dev/null +++ b/src/sec_certs/sample/cpe.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from typing import Any, ClassVar + +from sec_certs import constants +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.serialization.pandas import PandasSerializableType +from sec_certs.utils import helpers + + +@dataclass(init=False) +class CPE(PandasSerializableType, ComplexSerializableType): + uri: str + version: str + vendor: str + item_name: str + title: str | None + start_version: tuple[str, str] | None + end_version: tuple[str, str] | None + + __slots__ = ["uri", "version", "vendor", "item_name", "title", "start_version", "end_version"] + + pandas_columns: ClassVar[list[str]] = [ + "uri", + "vendor", + "item_name", + "version", + "title", + ] + + def __init__( + self, + uri: str, + title: str | None = None, + start_version: tuple[str, str] | None = None, + end_version: tuple[str, str] | None = None, + ): + super().__init__() + self.uri = uri + + splitted = helpers.split_unescape(self.uri, ":") + self.vendor = " ".join(splitted[3].split("_")) + self.item_name = " ".join(splitted[4].split("_")) + self.version = self.normalize_version(" ".join(splitted[5].split("_"))) + self.title = title + self.start_version = start_version + self.end_version = end_version + + def __lt__(self, other: CPE) -> bool: + return self.uri < other.uri + + @staticmethod + def normalize_version(version: str) -> str: + """ + Maps common empty versions (empty '', asterisk '*') to unified empty version (constants.CPE_VERSION_NA) + """ + if version in {"", "*"}: + return constants.CPE_VERSION_NA + return version + + @classmethod + def from_dict(cls, dct: dict[str, Any]) -> CPE: + if isinstance(dct["start_version"], list): + dct["start_version"] = tuple(dct["start_version"]) + if isinstance(dct["end_version"], list): + dct["end_version"] = tuple(dct["end_version"]) + return super().from_dict(dct) + + @property + def serialized_attributes(self) -> list[str]: + return ["uri", "title", "start_version", "end_version"] + + @property + def update(self) -> str: + if self.uri is None: + raise RuntimeError("URI is missing.") + return " ".join(self.uri.split(":")[6].split("_")) + + @property + def target_hw(self) -> str: + if self.uri is None: + raise RuntimeError("URI is missing.") + return " ".join(self.uri.split(":")[10].split("_")) + + @property + def pandas_tuple(self) -> tuple: + return self.uri, self.vendor, self.item_name, self.version, self.title + + def __hash__(self) -> int: + return hash((self.uri, self.start_version, self.end_version)) + + def __eq__(self, other: object) -> bool: + return isinstance(other, self.__class__) and self.uri == other.uri + + +@lru_cache(maxsize=4096) +def cached_cpe(*args, **kwargs): + return CPE(*args, **kwargs) diff --git a/src/sec_certs/sample/cve.py b/src/sec_certs/sample/cve.py new file mode 100644 index 00000000..ec024d35 --- /dev/null +++ b/src/sec_certs/sample/cve.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import datetime +import itertools +from dataclasses import dataclass +from typing import Any, ClassVar + +from dateutil.parser import isoparse + +from sec_certs.sample.cpe import CPE, cached_cpe +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.serialization.pandas import PandasSerializableType + + +@dataclass(init=False) +class CVE(PandasSerializableType, ComplexSerializableType): + @dataclass(eq=True) + class Impact(ComplexSerializableType): + base_score: float + severity: str + exploitability_score: float + impact_score: float + + __slots__ = ["base_score", "severity", "exploitability_score", "impact_score"] + + @classmethod + def from_nist_dict(cls, dct: dict[str, Any]) -> CVE.Impact: + """ + Will load Impact from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1 + """ + if not dct["impact"]: + return cls(0, "", 0, 0) + elif "baseMetricV3" in dct["impact"]: + return cls( + dct["impact"]["baseMetricV3"]["cvssV3"]["baseScore"], + dct["impact"]["baseMetricV3"]["cvssV3"]["baseSeverity"], + dct["impact"]["baseMetricV3"]["exploitabilityScore"], + dct["impact"]["baseMetricV3"]["impactScore"], + ) + elif "baseMetricV2" in dct["impact"]: + return cls( + dct["impact"]["baseMetricV2"]["cvssV2"]["baseScore"], + dct["impact"]["baseMetricV2"]["severity"], + dct["impact"]["baseMetricV2"]["exploitabilityScore"], + dct["impact"]["baseMetricV2"]["impactScore"], + ) + raise ValueError("NIST Dict for CVE Impact badly formatted.") + + cve_id: str + vulnerable_cpes: list[CPE] + impact: Impact + published_date: datetime.datetime | None + cwe_ids: set[str] | None + + __slots__ = ["cve_id", "vulnerable_cpes", "impact", "published_date", "cwe_ids"] + + pandas_columns: ClassVar[list[str]] = [ + "cve_id", + "vulnerable_cpes", + "base_score", + "severity", + "explotability_score", + "impact_score", + "published_date", + "cwe_ids", + ] + + def __init__( + self, cve_id: str, vulnerable_cpes: list[CPE], impact: Impact, published_date: str, cwe_ids: set[str] | None + ): + super().__init__() + self.cve_id = cve_id + self.vulnerable_cpes = vulnerable_cpes + self.impact = impact + self.published_date = isoparse(published_date) + self.cwe_ids = cwe_ids + + def __hash__(self) -> int: + return hash(self.cve_id) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, CVE): + return False + return self.cve_id == other.cve_id + + def __lt__(self, other: object) -> bool: + if not isinstance(other, CVE): + raise ValueError(f"Cannot compare CVE with {type(other)} type.") + self_year = int(self.cve_id.split("-")[1]) + self_id = int(self.cve_id.split("-")[2]) + other_year = int(other.cve_id.split("-")[1]) + other_id = int(other.cve_id.split("-")[2]) + + return self_year < other_year if self_year != other_year else self_id < other_id + + @property + def pandas_tuple(self): + return ( + self.cve_id, + self.vulnerable_cpes, + self.impact.base_score, + self.impact.severity, + self.impact.exploitability_score, + self.impact.impact_score, + self.published_date, + self.cwe_ids, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "cve_id": self.cve_id, + "vulnerable_cpes": self.vulnerable_cpes, + "impact": self.impact, + "published_date": self.published_date.isoformat() if self.published_date else None, + "cwe_ids": self.cwe_ids, + } + + @staticmethod + def _parse_nist_dict(lst: list) -> list[CPE]: + cpes: list[CPE] = [] + + for x in lst: + if x["vulnerable"]: + cpe_uri = x["cpe23Uri"] + version_start: tuple[str, str] | None + version_end: tuple[str, str] | None + if "versionStartIncluding" in x and x["versionStartIncluding"]: + version_start = ("including", x["versionStartIncluding"]) + elif "versionStartExcluding" in x and x["versionStartExcluding"]: + version_start = ("excluding", x["versionStartExcluding"]) + else: + version_start = None + + if "versionEndIncluding" in x and x["versionEndIncluding"]: + version_end = ("including", x["versionEndIncluding"]) + elif "versionEndExcluding" in x and x["versionEndExcluding"]: + version_end = ("excluding", x["versionEndExcluding"]) + else: + version_end = None + + cpes.append(cached_cpe(cpe_uri, start_version=version_start, end_version=version_end)) + + return cpes + + @classmethod + def from_nist_dict(cls, dct: dict) -> CVE: + """ + Will load CVE from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1 + """ + + def get_vulnerable_cpes_from_nist_dict(dct: dict) -> list[CPE]: + def get_vulnerable_cpes_from_node(node: dict) -> list[CPE]: + cpes: list[CPE] = [] + + if node["operator"] == "AND": + return cpes + + if "children" in node: + for child in node["children"]: + cpes += get_vulnerable_cpes_from_node(child) + + if "cpe_match" not in node: + return cpes + + candidates = node["cpe_match"] + cpes += CVE._parse_nist_dict(candidates) + + return cpes + + return list( + itertools.chain.from_iterable(get_vulnerable_cpes_from_node(x) for x in dct["configurations"]["nodes"]) + ) + + cve_id = dct["cve"]["CVE_data_meta"]["ID"] + impact = cls.Impact.from_nist_dict(dct) + vulnerable_cpes = get_vulnerable_cpes_from_nist_dict(dct) + published_date = dct["publishedDate"] + cwe_ids = cls.parse_cwe_data(dct) + + return cls(cve_id, vulnerable_cpes, impact, published_date, cwe_ids) + + @staticmethod + def parse_cwe_data(dct: dict) -> set[str] | None: + descriptions = dct["cve"]["problemtype"]["problemtype_data"][0]["description"] + return {x["value"] for x in descriptions} if descriptions else None diff --git a/src/sec_certs/sample/fips.py b/src/sec_certs/sample/fips.py new file mode 100644 index 00000000..aec791d4 --- /dev/null +++ b/src/sec_certs/sample/fips.py @@ -0,0 +1,654 @@ +from __future__ import annotations + +import itertools +import re +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Any, Callable, ClassVar, Final, Literal + +import dateutil +import numpy as np +import pandas as pd +import requests +from bs4 import BeautifulSoup, Tag +from tabula import read_pdf + +import sec_certs.constants as constants +import sec_certs.utils.extract +import sec_certs.utils.helpers as helpers +import sec_certs.utils.pdf +import sec_certs.utils.pdf as pdf +import sec_certs.utils.tables as tables +from sec_certs.cert_rules import FIPS_ALGS_IN_TABLE, fips_rules +from sec_certs.config.configuration import config +from sec_certs.sample.certificate import Certificate +from sec_certs.sample.certificate import Heuristics as BaseHeuristics +from sec_certs.sample.certificate import PdfData as BasePdfData +from sec_certs.sample.certificate import References, logger +from sec_certs.sample.cpe import CPE +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.serialization.pandas import PandasSerializableType +from sec_certs.utils.helpers import fips_dgst + + +class FIPSHTMLParser: + def __init__(self, soup: BeautifulSoup): + self._soup = soup + + def get_web_data_and_algorithms(self) -> tuple[set[str], FIPSCertificate.WebData]: + divs = self._soup.find_all("div", class_="panel panel-default") + details_div, vendor_div, related_files_div, validation_history_div = divs + details_dict = self._build_details_dict(details_div) + + vendor_dict = self._build_vendor_dict(vendor_div) + related_files_dict = self._build_related_files_dict(related_files_div) + validation_history_dict = self._build_validation_history_dict(validation_history_div) + + algorithms = set() + if "algorithms" in details_dict: + algorithms_data = details_dict.pop("algorithms") + for category, alg_ids in algorithms_data.items(): + algorithms |= {category + x for x in alg_ids} + + return algorithms, FIPSCertificate.WebData( + **{**details_dict, **vendor_dict, **related_files_dict, **validation_history_dict} + ) + + def _build_details_dict(self, details_div: Tag) -> dict[str, Any]: + def parse_single_detail_entry(key, entry): + normalized_key = DETAILS_KEY_NORMALIZATION_DICT[key] + normalization_func = DETAILS_KEY_TO_NORMALIZATION_FUNCTION.get(normalized_key, None) + normalized_entry = ( + FIPSHTMLParser.normalize_string(entry.text) if not normalization_func else normalization_func(entry) + ) + return normalized_key, normalized_entry + + entries = details_div.find_all("div", class_="row padrow") + entries = zip( + [x.find("div", class_="col-md-3") for x in entries], [x.find("div", class_="col-md-9") for x in entries] + ) + entries = [(FIPSHTMLParser.normalize_string(key.text), entry) for key, entry in entries] + entries = [parse_single_detail_entry(*x) for x in entries if x[0] in DETAILS_KEY_NORMALIZATION_DICT.keys()] + entries = {x: y for x, y in entries} + + if "caveat" in entries: + entries["mentioned_certs"] = FIPSHTMLParser.get_mentioned_certs_from_caveat(entries["caveat"]) + + # Temporarily disabled, as this isn't extracting anything useful. Only UNKNOWN#1-9 algs were extracted over whole dataset. + # if "description" in entries: + # algs = FIPSHTMLParser.get_algs_from_description(entries["description"]) + # if "algorithms" in entries: + # entries["algorithms"].update({"UNKNOWN": x for x in algs}) + # else: + # entries["algorithms"] = {"UNKNOWN": x for x in algs} + + return entries + + @staticmethod + def _build_vendor_dict(vendor_div: Tag) -> dict[str, Any]: + if not (link := vendor_div.find("a")): + return {"vendor_url": None, "vendor": list(vendor_div.find("div", "panel-body").children)[0].strip()} + else: + return {"vendor_url": link.get("href"), "vendor": link.text.strip()} + + @staticmethod + def _build_related_files_dict(related_files_div: Tag) -> dict[str, Any]: + if cert_link := [x for x in related_files_div.find_all("a") if "Certificate" in x.text]: + return {"certificate_pdf_url": constants.FIPS_BASE_URL + cert_link[0].get("href")} + else: + return {"certificate_pdf_url": None} + + @staticmethod + def _build_validation_history_dict(validation_history_div: Tag) -> dict[str, Any]: + def parse_row(row): + validation_date, validation_type, lab = row.find_all("td") + return FIPSCertificate.ValidationHistoryEntry( + dateutil.parser.parse(validation_date.text).date(), validation_type.text, lab.text + ) + + rows = validation_history_div.find("tbody").find_all("tr") + history: list[FIPSCertificate.ValidationHistoryEntry] | None = [parse_row(x) for x in rows] if rows else None + return {"validation_history": history} + + @staticmethod + def get_mentioned_certs_from_caveat(caveat: str) -> dict[str, int]: + ids_found: dict[str, int] = {} + r_key = r"(?P\w+)?\s?(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)+(?P\d+)" + for m in re.finditer(r_key, caveat): + if m.group("word") and m.group("word").lower() in {"rsa", "shs", "dsa", "pkcs", "aes"}: + continue + if m.group("id") in ids_found: + ids_found[m.group("id")] += 1 + else: + ids_found[m.group("id")] = 1 + return ids_found + + @staticmethod + def get_algs_from_description(description: str) -> set[str]: + return {m.group() for m in re.finditer(FIPS_ALGS_IN_TABLE, description)} + + @staticmethod + def parse_algorithms(algorithms_div: Tag) -> dict[str, set[str]]: + rows = algorithms_div.find("tbody").find_all("tr") + dct: dict[str, set[str]] = dict() + for row in rows: + cells = row.find_all("td") + dct[cells[0].text] = {m.group() for m in re.finditer(FIPS_ALGS_IN_TABLE, cells[1].text)} + return dct + + @staticmethod + def normalize_string(string: str) -> str: + return " ".join(string.split()) + + @staticmethod + def parse_tested_configurations(tested_configurations: Tag) -> list[str] | None: + configurations = [y.text for y in tested_configurations.find_all("li")] + return configurations if not configurations == ["N/A"] else None + + @staticmethod + def normalize_embodiment(embodiment_element: Tag) -> str: + text = FIPSHTMLParser.normalize_string(embodiment_element.text) + embodiment_normalization_dict = { + "Multi-chip embedded": "Multi-Chip Embedded", + "Multi-chip Standalone": "Multi-Chip Stand Alone", + "Multi-chip standalone": "Multi-Chip Stand Alone", + "Single-chip": "Single Chip", + } + return embodiment_normalization_dict.get(text, text) + + +DETAILS_KEY_NORMALIZATION_DICT: Final[dict[str, str]] = { + "Module Name": "module_name", + "Standard": "standard", + "Status": "status", + "Sunset Date": "date_sunset", + "Validation Dates": "date_validation", + "Overall Level": "level", + "Caveat": "caveat", + "Security Level Exceptions": "exceptions", + "Module Type": "module_type", + "Embodiment": "embodiment", + "Approved Algorithms": "algorithms", + "Tested Configuration(s)": "tested_conf", + "Description": "description", + "Historical Reason": "historical_reason", + "Hardware Versions": "hw_versions", + "Firmware Versions": "fw_versions", + "Revoked Reason": "revoked_reason", + "Revoked Link": "revoked_link", + "Software Versions": "sw_versions", + "Product URL": "product_url", +} + +DETAILS_KEY_TO_NORMALIZATION_FUNCTION: dict[str, Callable] = { + "date_sunset": lambda x: dateutil.parser.parse(x.text).date(), + "algorithms": getattr(FIPSHTMLParser, "parse_algorithms"), + "tested_conf": getattr(FIPSHTMLParser, "parse_tested_configurations"), + "exceptions": lambda x: [y.text for y in x.find_all("li")], + "status": lambda x: FIPSHTMLParser.normalize_string(x.text).lower(), + "level": lambda x: int(FIPSHTMLParser.normalize_string(x.text)), + "embodiment": getattr(FIPSHTMLParser, "normalize_embodiment"), +} + + +class FIPSCertificate( + Certificate["FIPSCertificate", "FIPSCertificate.Heuristics", "FIPSCertificate.PdfData"], + PandasSerializableType, + ComplexSerializableType, +): + """ + Data structure for common FIPS 140 certificate. Contains several inner classes that layer the data logic. + Can be serialized into/from json (`ComplexSerializableType`). + Is basic element of `FIPSDataset`. The functionality is mostly related to holding data and transformations that + the certificate can handle itself. `FIPSDataset` class then instrument this functionality. + """ + + pandas_columns: ClassVar[list[str]] = [ + "dgst", + "cert_id", + "name", + "status", + "standard", + "type", + "level", + "embodiment", + "date_validation", + "date_sunset", + "algorithms", + "extracted_versions", + "cpe_matches", + "verified_cpe_matches", + "related_cves", + "module_directly_referenced_by", + "module_indirectly_referenced_by", + "module_directly_referencing", + "module_indirectly_referencing", + "policy_directly_referenced_by", + "policy_indirectly_referenced_by", + "policy_directly_referencing", + "policy_indirectly_referencing", + ] + + @dataclass(eq=True) + class InternalState(ComplexSerializableType): + """ + Holds state of the `FIPSCertificate` + """ + + module_download_ok: bool + policy_download_ok: bool + + policy_convert_garbage: bool + policy_convert_ok: bool + + module_extract_ok: bool + policy_extract_ok: bool + + policy_pdf_hash: str | None + policy_txt_hash: str | None + + policy_pdf_path: Path + policy_txt_path: Path + module_html_path: Path + + def __init__( + self, + module_download_ok: bool = False, + policy_download_ok: bool = False, + policy_convert_garbage: bool = False, + policy_convert_ok: bool = False, + module_extract_ok: bool = False, + policy_extract_ok: bool = False, + policy_pdf_hash: str | None = None, + policy_txt_hash: str | None = None, + ): + self.module_download_ok = module_download_ok + self.policy_download_ok = policy_download_ok + self.policy_convert_garbage = policy_convert_garbage + self.policy_convert_ok = policy_convert_ok + self.module_extract_ok = module_extract_ok + self.policy_extract_ok = policy_extract_ok + self.policy_pdf_hash = policy_pdf_hash + self.policy_txt_hash = policy_txt_hash + + @property + def serialized_attributes(self) -> list[str]: + return [ + "module_download_ok", + "policy_download_ok", + "policy_convert_garbage", + "policy_convert_ok", + "module_extract_ok", + "policy_extract_ok", + "policy_pdf_hash", + "policy_txt_hash", + ] + + def module_is_ok_to_download(self, fresh: bool = True) -> bool: + return True if fresh else not self.module_download_ok + + def policy_is_ok_to_download(self, fresh: bool = True) -> bool: + return True if fresh else not self.policy_download_ok + + def policy_is_ok_to_convert(self, fresh: bool = True) -> bool: + return self.policy_download_ok if fresh else self.policy_download_ok and not self.policy_convert_ok + + def module_is_ok_to_analyze(self, fresh: bool = True) -> bool: + return ( + self.module_download_ok and self.module_extract_ok + if fresh + else self.module_download_ok and not self.module_extract_ok + ) + + def policy_is_ok_to_analyze(self, fresh: bool = True) -> bool: + return ( + self.policy_convert_ok and self.policy_extract_ok + if fresh + else self.policy_convert_ok and not self.policy_extract_ok + ) + + def set_local_paths(self, policies_pdf_dir: Path, policies_txt_dir: Path, modules_html_dir: Path) -> None: + self.state.policy_pdf_path = (policies_pdf_dir / str(self.dgst)).with_suffix(".pdf") + self.state.policy_txt_path = (policies_txt_dir / str(self.dgst)).with_suffix(".txt") + self.state.module_html_path = (modules_html_dir / str(self.dgst)).with_suffix(".html") + + @dataclass(eq=True) + class ValidationHistoryEntry(ComplexSerializableType): + date: date + validation_type: Literal["initial", "update"] + lab: str + + @classmethod + def from_dict(cls, dct: dict) -> FIPSCertificate.ValidationHistoryEntry: + new_dct = dct.copy() + new_dct["date"] = dateutil.parser.parse(dct["date"]).date() + return cls(**new_dct) + + @dataclass(eq=True) + class WebData(ComplexSerializableType): + """ + Data structure for data obtained from scanning certificate webpage at NIST.gov + """ + + module_name: str | None = field(default=None) + validation_history: list[FIPSCertificate.ValidationHistoryEntry] | None = field(default=None) + vendor_url: str | None = field(default=None) + vendor: str | None = field(default=None) + certificate_pdf_url: str | None = field(default=None) + module_type: str | None = field(default=None) + standard: str | None = field(default=None) + status: Literal["active", "historical", "revoked"] | None = field(default=None) + level: Literal[1, 2, 3, 4] | None = field(default=None) + caveat: str | None = field(default=None) + exceptions: list[str] | None = field(default=None) + embodiment: str | None = field(default=None) + description: str | None = field(default=None) + tested_conf: list[str] | None = field(default=None) + hw_versions: str | None = field(default=None) + fw_versions: str | None = field(default=None) + sw_versions: str | None = field(default=None) + mentioned_certs: dict[str, int] | None = field(default=None) # Cert_id: n_occurences + historical_reason: str | None = field(default=None) + date_sunset: date | None = field(default=None) + revoked_reason: str | None = field(default=None) + revoked_link: str | None = field(default=None) + + # Those below are left unused at the moment + # product_url: Optional[str] = field(default=None) + + def __repr__(self) -> str: + return ( + self.module_name + if self.module_name is not None + else "" + " created by " + self.vendor + if self.vendor is not None + else "" + ) + + def __str__(self) -> str: + return repr(self) + + @classmethod + def from_dict(cls, dct: dict) -> FIPSCertificate.WebData: + new_dct = dct.copy() + if new_dct["date_sunset"]: + new_dct["date_sunset"] = dateutil.parser.parse(new_dct["date_sunset"]).date() + return cls(**dct) + + @dataclass(eq=True) + class PdfData(BasePdfData, ComplexSerializableType): + """ + Data structure that holds data obtained from scanning pdf files (or their converted txt documents). + """ + + keywords: dict = field(default_factory=dict) + policy_metadata: dict[str, Any] = field(default_factory=dict) + + @property + def certlike_algorithm_numbers(self) -> set[str]: + """Returns numbers of certificates from keywords["fips_certlike"]["Certlike"]""" + if self.keywords and "fips_certlike" in self.keywords: + fips_certlike = self.keywords["fips_certlike"].get("Certlike", dict()) + matches = {re.search(r"#\s{0,1}\d{1,4}", x) for x in fips_certlike.keys()} + return {"".join([x for x in match.group() if x.isdigit()]) for match in matches if match} + else: + return set() + + @dataclass(eq=True) + class Heuristics(BaseHeuristics, ComplexSerializableType): + """ + Data structure that holds data obtained by processing the certificate and applying various heuristics. + """ + + algorithms: set[str] = field(default_factory=set) + extracted_versions: set[str] = field(default_factory=set) + cpe_matches: set[str] | None = field(default=None) + verified_cpe_matches: set[CPE] | None = field(default=None) + related_cves: set[str] | None = field(default=None) + policy_prunned_references: set[str] = field(default_factory=set) + module_prunned_references: set[str] = field(default_factory=set) + policy_processed_references: References = field(default_factory=References) + module_processed_references: References = field(default_factory=References) + direct_transitive_cves: set[str] | None = field(default=None) + indirect_transitive_cves: set[str] | None = field(default=None) + + @property + def algorithm_numbers(self) -> set[str]: + """Returns numbers of algorithms""" + + def alg_to_number(alg: str) -> str: + return "".join([x for x in alg.split("#")[1] if x.isdigit()]) + + return {alg_to_number(x) for x in self.algorithms} + + @property + def dgst(self) -> str: + """ + Returns primary key of the certificate, its id. + """ + return fips_dgst(self.cert_id) + + @property + def manufacturer(self) -> str | None: # type: ignore + return self.web_data.vendor + + @property + def module_html_url(self) -> str: + return constants.FIPS_MODULE_URL.format(self.cert_id) + + @property + def policy_pdf_url(self) -> str: + return constants.FIPS_SP_URL.format(self.cert_id) + + @property + def name(self) -> str | None: # type: ignore + return self.web_data.module_name + + @property + def label_studio_title(self) -> str: + return ( + "Vendor: " + + str(self.web_data.vendor) + + "\n" + + "Module name: " + + str(self.web_data.module_name) + + "\n" + + "HW version: " + + str(self.web_data.hw_versions) + + "\n" + + "FW version: " + + str(self.web_data.fw_versions) + ) + + def __init__( + self, + cert_id: str, + web_data: FIPSCertificate.WebData | None = None, + pdf_data: FIPSCertificate.PdfData | None = None, + heuristics: FIPSCertificate.Heuristics | None = None, + state: InternalState | None = None, + ): + super().__init__() + + self.cert_id = cert_id + self.web_data: FIPSCertificate.WebData = web_data if web_data else FIPSCertificate.WebData() + self.pdf_data: FIPSCertificate.PdfData = pdf_data if pdf_data else FIPSCertificate.PdfData() + self.heuristics: FIPSCertificate.Heuristics = heuristics if heuristics else FIPSCertificate.Heuristics() + self.state: FIPSCertificate.InternalState = state if state else FIPSCertificate.InternalState() + + @property + def pandas_tuple(self) -> tuple: + return ( + self.dgst, + self.cert_id, + self.web_data.module_name, + self.web_data.status, + self.web_data.standard, + self.web_data.module_type, + self.web_data.level, + self.web_data.embodiment, + self.web_data.validation_history[0].date if self.web_data.validation_history else np.nan, + self.web_data.date_sunset, + self.heuristics.algorithms, + self.heuristics.extracted_versions, + self.heuristics.cpe_matches, + self.heuristics.verified_cpe_matches, + self.heuristics.related_cves, + self.heuristics.module_processed_references.directly_referenced_by, + self.heuristics.module_processed_references.indirectly_referenced_by, + self.heuristics.module_processed_references.directly_referencing, + self.heuristics.module_processed_references.indirectly_referencing, + self.heuristics.policy_processed_references.directly_referenced_by, + self.heuristics.policy_processed_references.indirectly_referenced_by, + self.heuristics.policy_processed_references.directly_referencing, + self.heuristics.policy_processed_references.indirectly_referencing, + ) + + @staticmethod + def parse_html_module(cert: FIPSCertificate) -> FIPSCertificate: + with cert.state.module_html_path.open("r") as handle: + soup = BeautifulSoup(handle, "html5lib") + + parser = FIPSHTMLParser(soup) + algorithms, cert.web_data = parser.get_web_data_and_algorithms() + cert.heuristics.algorithms |= algorithms + cert.state.module_extract_ok = True + + return cert + + @staticmethod + def download_module(cert: FIPSCertificate) -> FIPSCertificate: + if (exit_code := helpers.download_file(cert.module_html_url, cert.state.module_html_path)) != requests.codes.ok: + error_msg = f"failed to download html module from {cert.module_html_url}, code {exit_code}" + logger.error(f"Cert dgst: {cert.dgst} " + error_msg) + cert.state.module_download_ok = False + else: + cert.state.module_download_ok = True + return cert + + @staticmethod + def download_policy(cert: FIPSCertificate) -> FIPSCertificate: + if (exit_code := helpers.download_file(cert.policy_pdf_url, cert.state.policy_pdf_path)) != requests.codes.ok: + error_msg = f"failed to download pdf policy from {cert.policy_pdf_url}, code {exit_code}" + logger.error(f"Cert dgst: {cert.dgst} " + error_msg) + cert.state.policy_download_ok = False + else: + cert.state.policy_download_ok = True + cert.state.policy_pdf_hash = helpers.get_sha256_filepath(cert.state.policy_pdf_path) + return cert + + @staticmethod + def convert_policy_pdf(cert: FIPSCertificate) -> FIPSCertificate: + """ + Converts policy pdf -> txt + """ + ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file( + cert.state.policy_pdf_path, cert.state.policy_txt_path + ) + + # If OCR was done and the result was garbage + cert.state.policy_convert_garbage = ocr_done + # And put the whole result into convert_ok + cert.state.policy_convert_ok = ok_result + + if not ok_result: + error_msg = "Failed to convert policy pdf->txt" + logger.error(f"Cert dgst: {cert.dgst}" + error_msg) + else: + cert.state.policy_txt_hash = helpers.get_sha256_filepath(cert.state.policy_txt_path) + + return cert + + @staticmethod + def extract_policy_pdf_metadata(cert: FIPSCertificate) -> FIPSCertificate: + """ + Extract the PDF metadata from the security policy. + """ + _, metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.policy_pdf_path) + + if metadata: + cert.pdf_data.policy_metadata = metadata + else: + cert.pdf_data.policy_metadata = dict() + cert.state.policy_extract_ok = False + return cert + + @staticmethod + def extract_policy_pdf_keywords(cert: FIPSCertificate) -> FIPSCertificate: + """ + Extract keywords from policy document + """ + keywords = sec_certs.utils.extract.extract_keywords(cert.state.policy_txt_path, fips_rules) + if not keywords: + cert.state.policy_extract_ok = False + else: + cert.pdf_data.keywords = keywords + return cert + + @staticmethod + def get_algorithms_from_policy_tables(cert: FIPSCertificate): + """ + Retrieves IDs of algorithms from tables inside security policy pdfs. + External library is used to handle this. + """ + if table_rich_page_numbers := tables.find_pages_with_tables(cert.state.policy_txt_path): + pdf.repair_pdf(cert.state.policy_pdf_path) + try: + tabular_data = read_pdf(cert.state.policy_pdf_path, pages=list(table_rich_page_numbers), silent=True) + cert.heuristics.algorithms |= set( + itertools.chain.from_iterable( + tables.get_algs_from_table(df.to_string()) + for df in tabular_data + if isinstance(df, pd.DataFrame) + ) + ) + except Exception as e: + logger.warning(f"Error when parsing tables from {cert.dgst}: {e}") + cert.state.policy_extract_ok = False + + def prune_referenced_cert_ids(self) -> None: + """ + This method goes through all IDs (numbers) that correspond to FIPS Certificates and are stored in + pdf_data.keywords or web_data.mentioned_certs. It performs prunning of these attributes and fills attributes + heuristics.prunned_module_references and heuristics.prunned_policy_references. These variables are further + processed and Reference objects are created from them. + """ + html_module_ids = set(self.web_data.mentioned_certs.keys()) if self.web_data.mentioned_certs else set() + self.heuristics.module_prunned_references = self._prune_reference_ids_variable(html_module_ids) + + if self.pdf_data.keywords: + pdf_policy_ids = set(self.pdf_data.keywords["fips_cert_id"].get("Cert", dict()).keys()) + pdf_policy_ids = {"".join([y for y in x if y.isdigit()]) for x in pdf_policy_ids} + else: + pdf_policy_ids = set() + + self.heuristics.policy_prunned_references = self._prune_reference_ids_variable(pdf_policy_ids) + + def compute_heuristics_version(self) -> None: + """ + Heuristically computes the version of the product. + """ + versions_for_extraction = "" + if self.web_data.module_name: + versions_for_extraction += f" {self.web_data.module_name}" + if self.web_data.hw_versions: + versions_for_extraction += f" {self.web_data.hw_versions}" + if self.web_data.fw_versions: + versions_for_extraction += f" {self.web_data.fw_versions}" + self.heuristics.extracted_versions = helpers.compute_heuristics_version(versions_for_extraction) + + def _prune_reference_ids_variable(self, attribute_to_prune: set[str]) -> set[str]: + """ + Prunnes cert_ids from variable "attribute_to_prune", return result. Steps: + 0. Consider only ids != self.cert_id + 1. Consider only ids > config.always_false_positive_fips_cert_id_threshold + 2. Consider only ids s.t. they don't appear in self.heuristics.algorithms + 3. Consider only ids s.t. they don't appear in self.pdf_data.keywords["fips_certlike"]["Certlike"] + """ + prunned = {x for x in attribute_to_prune if x != self.cert_id} + prunned = {x for x in prunned if int(x) > config.always_false_positive_fips_cert_id_threshold} + prunned = {x for x in prunned if x not in self.heuristics.algorithm_numbers} + prunned = {x for x in prunned if x not in self.pdf_data.certlike_algorithm_numbers} + + return prunned diff --git a/src/sec_certs/sample/fips_algorithm.py b/src/sec_certs/sample/fips_algorithm.py new file mode 100644 index 00000000..16f19e64 --- /dev/null +++ b/src/sec_certs/sample/fips_algorithm.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date +from typing import ClassVar + +from sec_certs import constants +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.serialization.pandas import PandasSerializableType + + +@dataclass(eq=True, frozen=True) +class FIPSAlgorithm(PandasSerializableType, ComplexSerializableType): + """ + Data structure for algorithm of `FIPSCertificate` + """ + + alg_number: str + algorithm_type: str + vendor: str + implementation_name: str + validation_date: date + + pandas_columns: ClassVar[list[str]] = [ + "dgst", + "alg_number", + "algorithm_type", + "vendor", + "implementation_name", + "validation_date", + ] + + @property + def pandas_tuple(self) -> tuple: + return ( + self.dgst, + self.alg_number, + self.algorithm_type, + self.vendor, + self.implementation_name, + self.validation_date, + ) + + @property + def dgst(self) -> str: + return f"{self.algorithm_type}{self.alg_number}" + + @property + def page_url(self) -> str: + return constants.FIPS_ALG_URL.format(self.algorithm_type, self.alg_number) diff --git a/src/sec_certs/sample/fips_iut.py b/src/sec_certs/sample/fips_iut.py new file mode 100644 index 00000000..cb521ee4 --- /dev/null +++ b/src/sec_certs/sample/fips_iut.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date, datetime +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Iterator, Mapping + +import requests +from bs4 import BeautifulSoup, Tag + +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.utils.helpers import to_utc + + +@dataclass(frozen=True) +class IUTEntry(ComplexSerializableType): + module_name: str + vendor_name: str + standard: str + iut_date: date + + def to_dict(self) -> dict[str, str]: + return {**self.__dict__, "iut_date": self.iut_date.isoformat()} + + @classmethod + def from_dict(cls, dct: Mapping) -> IUTEntry: + return cls( + dct["module_name"], + dct["vendor_name"], + dct["standard"], + date.fromisoformat(dct["iut_date"]), + ) + + +@dataclass +class IUTSnapshot(ComplexSerializableType): + entries: set[IUTEntry] + timestamp: datetime + last_updated: date + displayed: int | None + not_displayed: int | None + total: int | None + + def __len__(self) -> int: + return len(self.entries) + + def __iter__(self) -> Iterator[IUTEntry]: + yield from self.entries + + def to_dict(self) -> dict[str, int | None | list[IUTEntry] | str]: + return { + "entries": list(self.entries), + "timestamp": self.timestamp.isoformat(), + "last_updated": self.last_updated.isoformat(), + "displayed": self.displayed, + "not_displayed": self.not_displayed, + "total": self.total, + } + + @classmethod + def from_dict(cls, dct: Mapping) -> IUTSnapshot: + return cls( + set(dct["entries"]), + datetime.fromisoformat(dct["timestamp"]), + date.fromisoformat(dct["last_updated"]), + dct["displayed"], + dct["not_displayed"], + dct["total"], + ) + + @classmethod + def from_page(cls, content: bytes, snapshot_date: datetime) -> IUTSnapshot: + """ + Get an IUT snapshot from a HTML dump of the FIPS website. + """ + if not content: + raise ValueError("Empty content in IUT.") + soup = BeautifulSoup(content, "html5lib") + tables = soup.find_all("table") + if len(tables) != 1: + raise ValueError("Not only a single table in IUT.") + + last_updated_elem = next( + filter( + lambda e: isinstance(e, Tag) and e.name == "p", + soup.find(id="content").next_siblings, + ) + ) + last_updated_text = str(last_updated_elem.string).strip() + last_updated = datetime.strptime(last_updated_text, "Last Updated: %m/%d/%Y").date() + table = tables[0].find("tbody") + lines = table.find_all("tr") + entries = { + IUTEntry( + str(line[0].string), + str(line[1].string), + str(line[2].string), + datetime.strptime(str(line[3].string), "%m/%d/%Y").date(), + ) + for line in map(lambda tr: tr.find_all("td"), lines) + } + + # Parse footer + footer = soup.find(id="IUTFooter") + displayed: int | None + not_displayed: int | None + total: int | None + + if footer: + footer_lines = footer.find_all("tr") + displayed = int(footer_lines[0].find_all("td")[1].text) + not_displayed = int(footer_lines[1].find_all("td")[1].text) + total = int(footer_lines[2].find_all("td")[1].text) + else: + displayed, not_displayed, total = (None, None, None) + + return cls( + entries=entries, + timestamp=snapshot_date, + last_updated=last_updated, + displayed=displayed, + not_displayed=not_displayed, + total=total, + ) + + @classmethod + def from_dump(cls, dump_path: str | Path, snapshot_date: datetime | None = None) -> IUTSnapshot: + """ + Get an IUT snapshot from a HTML file dump of the FIPS website. + """ + dump_path = Path(dump_path) + if snapshot_date is None: + try: + snapshot_date = to_utc(datetime.fromisoformat(dump_path.name[len("fips_iut_") : -len(".html")])) + except Exception: + raise ValueError("snapshot_date not given and could not be inferred from filename.") + with dump_path.open("rb") as f: + content = f.read() + return cls.from_page(content, snapshot_date) + + @classmethod + def from_web(cls) -> IUTSnapshot: + """ + Get an IUT snapshot from the FIPS website right now. + """ + iut_resp = requests.get(constants.FIPS_IUT_URL) + if iut_resp.status_code != 200: + raise ValueError(f"Getting IUT snapshot failed: {iut_resp.status_code}") + + snapshot_date = to_utc(datetime.now()) + return cls.from_page(iut_resp.content, snapshot_date) + + @classmethod + def from_web_latest(cls) -> IUTSnapshot: + """ + Get a IUT snapshot from seccerts.org. + """ + iut_resp = requests.get(config.fips_iut_latest_snapshot) + if iut_resp.status_code != 200: + raise ValueError(f"Getting MIP snapshot failed: {iut_resp.status_code}") + with NamedTemporaryFile() as tmpfile: + tmpfile.write(iut_resp.content) + return cls.from_json(tmpfile.name) diff --git a/src/sec_certs/sample/fips_mip.py b/src/sec_certs/sample/fips_mip.py new file mode 100644 index 00000000..6918d2aa --- /dev/null +++ b/src/sec_certs/sample/fips_mip.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import date, datetime +from enum import Enum +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Iterator, Mapping + +import requests +from bs4 import BeautifulSoup, Tag + +from sec_certs import constants +from sec_certs.config.configuration import config +from sec_certs.constants import FIPS_MIP_STATUS_RE +from sec_certs.serialization.json import ComplexSerializableType +from sec_certs.utils.helpers import to_utc + +logger = logging.getLogger(__name__) + + +class MIPStatus(Enum): + IN_REVIEW = "In Review" + REVIEW_PENDING = "Review Pending" + COORDINATION = "Coordination" + FINALIZATION = "Finalization" + + +@dataclass(frozen=True) +class MIPEntry(ComplexSerializableType): + module_name: str + vendor_name: str + standard: str + status: MIPStatus | None + status_since: date | None + + def to_dict(self) -> dict[str, str | MIPStatus | None | date | None]: + return { + **self.__dict__, + "status": self.status.value if self.status else None, + "status_since": self.status_since.isoformat() if self.status_since else None, + } + + @classmethod + def from_dict(cls, dct: Mapping) -> MIPEntry: + return cls( + dct["module_name"], + dct["vendor_name"], + dct["standard"], + MIPStatus(dct["status"]) if dct["status"] else None, + date.fromisoformat(dct["status_since"]) if dct.get("status_since") else None, + ) + + +@dataclass +class MIPSnapshot(ComplexSerializableType): + entries: set[MIPEntry] + timestamp: datetime + last_updated: date + displayed: int + not_displayed: int + total: int + + def __len__(self) -> int: + return len(self.entries) + + def __iter__(self) -> Iterator[MIPEntry]: + yield from self.entries + + def to_dict(self) -> dict[str, int | str | list[MIPEntry]]: + return { + "entries": list(self.entries), + "timestamp": self.timestamp.isoformat(), + "last_updated": self.last_updated.isoformat(), + "displayed": self.displayed, + "not_displayed": self.not_displayed, + "total": self.total, + } + + @classmethod + def from_dict(cls, dct: Mapping) -> MIPSnapshot: + return cls( + set(dct["entries"]), + datetime.fromisoformat(dct["timestamp"]), + date.fromisoformat(dct["last_updated"]), + dct["displayed"], + dct["not_displayed"], + dct["total"], + ) + + @classmethod + def _extract_entries_1(cls, lines): + """Works until 2020.10.28 (including).""" + entries = set() + for tr in lines: + tds = tr.find_all("td") + status = None + if "mip-highlight" in tds[-1]["class"]: + status = MIPStatus.FINALIZATION + elif "mip-highlight" in tds[-2]["class"]: + status = MIPStatus.COORDINATION + elif "mip-highlight" in tds[-3]["class"]: + status = MIPStatus.REVIEW_PENDING + elif "mip-highlight" in tds[-4]["class"]: + status = MIPStatus.IN_REVIEW + entries.add(MIPEntry(str(tds[0].string), str(tds[1].string), str(tds[2].string), status, None)) + return entries + + @classmethod + def _extract_entries_2(cls, lines): + """Works until 2021.04.20 (including).""" + return { + MIPEntry( + str(line[0].string), str(line[1].string), str(line[2].string), MIPStatus(str(line[3].string)), None + ) + for line in map(lambda tr: tr.find_all("td"), lines) + } + + @classmethod + def _extract_entries_3(cls, lines): + """Works until 2022.03.23 (including).""" + return { + MIPEntry( + str(line[0].string), + str(" ".join(line[1].find_all(text=True, recursive=False)).strip()), + str(line[2].string), + MIPStatus(str(line[3].string)), + None, + ) + for line in map(lambda tr: tr.find_all("td"), lines) + } + + @classmethod + def _extract_entries_4(cls, lines): + """Works now.""" + entries = set() + for line in map(lambda tr: tr.find_all("td"), lines): + module_name = str(line[0].string) + vendor_name = str(" ".join(line[1].find_all(text=True, recursive=False)).strip()) + standard = str(line[2].string) + status_line = FIPS_MIP_STATUS_RE.match(str(line[3].string)) + if status_line is None: + raise ValueError("Cannot parse MIP status line.") + status = MIPStatus(status_line.group("status")) + since = datetime.strptime(status_line.group("since"), "%m/%d/%Y").date() + entries.add(MIPEntry(module_name, vendor_name, standard, status, since)) + return entries + + @classmethod + def _extract_entries(cls, lines, snapshot_date): + if snapshot_date <= datetime(2020, 10, 28): + entries = cls._extract_entries_1(lines) + elif snapshot_date <= datetime(2021, 4, 20): + entries = cls._extract_entries_2(lines) + elif snapshot_date <= datetime(2022, 3, 23): + entries = cls._extract_entries_3(lines) + else: + entries = cls._extract_entries_4(lines) + return entries + + @classmethod + def from_page(cls, content: bytes, snapshot_date: datetime) -> MIPSnapshot: + """ + Get a MIP snapshot from a HTML dump of the FIPS website. + """ + if not content: + raise ValueError("Empty content in MIP.") + soup = BeautifulSoup(content, "html5lib") + tables = soup.find_all("table") + if len(tables) != 1: + raise ValueError("Not only a single table in MIP data.") + + # Parse Last Updated + last_updated_elem = next( + filter( + lambda e: isinstance(e, Tag) and e.name == "p", + soup.find(id="content").next_siblings, + ) + ) + last_updated_text = str(last_updated_elem.string).strip() + last_updated = datetime.strptime(last_updated_text, "Last Updated: %m/%d/%Y").date() + + # Parse entries + table = tables[0].find("tbody") + lines = table.find_all("tr") + entries = cls._extract_entries(lines, snapshot_date) + + # Parse footer + footer = soup.find(id="MIPFooter") + footer_lines = footer.find_all("tr") + displayed = int(footer_lines[0].find_all("td")[1].text) + not_displayed = int(footer_lines[1].find_all("td")[1].text) + total = int(footer_lines[2].find_all("td")[1].text) + + return cls( + entries=entries, + timestamp=snapshot_date, + last_updated=last_updated, + displayed=displayed, + not_displayed=not_displayed, + total=total, + ) + + @classmethod + def from_dump(cls, dump_path: str | Path, snapshot_date: datetime | None = None) -> MIPSnapshot: + """ + Get a MIP snapshot from a HTML file dump of the FIPS website. + """ + dump_path = Path(dump_path) + if snapshot_date is None: + try: + snapshot_date = to_utc(datetime.fromisoformat(dump_path.name[len("fips_mip_") : -len(".html")])) + except Exception: + raise ValueError("snapshot_date not given and could not be inferred from filename.") + with dump_path.open("rb") as f: + content = f.read() + return cls.from_page(content, snapshot_date) + + @classmethod + def from_web(cls) -> MIPSnapshot: + """ + Get a MIP snapshot from the FIPS website right now. + """ + mip_resp = requests.get(constants.FIPS_MIP_URL) + if mip_resp.status_code != 200: + raise ValueError(f"Getting MIP snapshot failed: {mip_resp.status_code}") + + snapshot_date = to_utc(datetime.now()) + return cls.from_page(mip_resp.content, snapshot_date) + + @classmethod + def from_web_latest(cls) -> MIPSnapshot: + """ + Get a MIP snapshot from seccerts.org. + """ + mip_resp = requests.get(config.fips_mip_latest_snapshot) + if mip_resp.status_code != 200: + raise ValueError(f"Getting MIP snapshot failed: {mip_resp.status_code}") + with NamedTemporaryFile() as tmpfile: + tmpfile.write(mip_resp.content) + return cls.from_json(tmpfile.name) diff --git a/src/sec_certs/sample/protection_profile.py b/src/sec_certs/sample/protection_profile.py new file mode 100644 index 00000000..b7c2ec34 --- /dev/null +++ b/src/sec_certs/sample/protection_profile.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import copy +import logging +from dataclasses import dataclass +from typing import Any + +import sec_certs.utils.sanitization as sanitization +from sec_certs.serialization.json import ComplexSerializableType + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ProtectionProfile(ComplexSerializableType): + """ + Object for holding protection profiles. + """ + + pp_name: str + pp_eal: str | None + pp_link: str | None = None + pp_ids: frozenset[str] | None = None + + def __post_init__(self): + super().__setattr__("pp_name", sanitization.sanitize_string(self.pp_name)) + super().__setattr__("pp_link", sanitization.sanitize_link(self.pp_link)) + + @classmethod + def from_dict(cls, dct: dict[str, Any]) -> ProtectionProfile: + new_dct = copy.deepcopy(dct) + new_dct["pp_ids"] = frozenset(new_dct["pp_ids"]) if new_dct["pp_ids"] else None + return cls(*tuple(new_dct.values())) + + @classmethod + def from_old_api_dict(cls, dct: dict[str, Any]) -> ProtectionProfile: + pp_name = sanitization.sanitize_string(dct["csv_scan"]["cc_pp_name"]) + pp_link = sanitization.sanitize_link(dct["csv_scan"]["link_pp_document"]) + pp_ids = frozenset(dct["processed"]["cc_pp_csvid"]) if dct["processed"]["cc_pp_csvid"] else None + eal_set = sanitization.sanitize_security_levels(dct["csv_scan"]["cc_security_level"]) + + if not len(eal_set) <= 1: + raise ValueError("EAL field should have single value or should be empty.") + + eal_str = list(eal_set)[0] if eal_set else None + + return cls(pp_name, eal_str, pp_link, pp_ids) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ProtectionProfile): + return False + return self.pp_name == other.pp_name and self.pp_link == other.pp_link + + def __lt__(self, other: ProtectionProfile) -> bool: + return self.pp_name < other.pp_name diff --git a/src/sec_certs/sample/sar.py b/src/sec_certs/sample/sar.py new file mode 100644 index 00000000..31359299 --- /dev/null +++ b/src/sec_certs/sample/sar.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from sec_certs.serialization.json import ComplexSerializableType + +SAR_CLASS_MAPPING = { + "APE": "Protection Profile evaluation", + "ACE": "Protection Profile configuration evaluation", + "ASE": "Security Target evaluation", + "ADV": "Development", + "AGD": "Guidance documents", + "ALC": "Life-cycle support", + "ATE": "Tests", + "AVA": "Vulnerability assessment", + "ACO": "Comoposition", +} + +SAR_CLASSES = {x for x in SAR_CLASS_MAPPING} +SAR_DICT_KEY = "cc_sar" + + +@dataclass(frozen=True, eq=True) +class SAR(ComplexSerializableType): + family: str + level: int + + @property + def assurance_class(self): + return SAR_CLASS_MAPPING.get(self.family.split("_")[0], None) + + @classmethod + def from_string(cls, string: str) -> SAR: + if not cls.contains_level(string): + raise ValueError("SAR misses level integer") + if not cls.matches_re(string): + raise ValueError("SAR does not match any regular expression") + family = string.split(".")[0] + level = int(string.split(".")[1]) + return cls(family, level) + + @staticmethod + def contains_level(string: str) -> bool: + if len(string.split(".")) == 1: + return False + return True + + @staticmethod + def matches_re(string: str) -> bool: + return any( + [re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING] + ) + + def __lt__(self, other: Any) -> bool: + if not isinstance(other, SAR): + raise ValueError(f"cannot compare {type(other)} with SAR.") + return str(self) < str(other) diff --git a/src/sec_certs/serialization/__init__.py b/src/sec_certs/serialization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sec_certs/serialization/json.py b/src/sec_certs/serialization/json.py new file mode 100644 index 00000000..69bbc2ff --- /dev/null +++ b/src/sec_certs/serialization/json.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import copy +import json +from datetime import date +from functools import wraps +from pathlib import Path +from typing import Any, Callable, TypeVar + +from sec_certs import constants + +T = TypeVar("T", bound="ComplexSerializableType") + + +class SerializationError(Exception): + pass + + +class ComplexSerializableType: + __slots__: tuple[str] + + def __init__(self, *args, **kwargs): + pass + + # Ideally, the serialized_fields would be an class variable referencing itself, but that it virtually impossible + # to achieve without using metaclasses. Not to complicate the code, we choose instance variable. + @property + def serialized_attributes(self) -> list[str]: + if hasattr(self, "__slots__") and self.__slots__: + return list(self.__slots__) + return list(self.__dict__.keys()) + + def to_dict(self) -> dict[str, Any]: + if hasattr(self, "__slots__") and self.__slots__: + return { + key: copy.deepcopy(getattr(self, key)) for key in self.__slots__ if key in self.serialized_attributes + } + return {key: val for key, val in copy.deepcopy(self.__dict__).items() if key in self.serialized_attributes} + + @classmethod + def from_dict(cls: type[T], dct: dict) -> T: + try: + return cls(**dct) + except TypeError as e: + raise TypeError(f"Dict: {dct} on {cls.__mro__}") from e + + def to_json(self, output_path: str | Path | None = None) -> None: + if not output_path and (not hasattr(self, "json_path") or not self.json_path): # type: ignore + raise SerializationError( + f"The object {self} of type {self.__class__} does not have json_path attribute set but to_json() was called without an argument." + ) + if not output_path: + output_path = self.json_path # type: ignore + if self.json_path == constants.DUMMY_NONEXISTING_PATH: # type: ignore + raise SerializationError(f"json_path attribute for '{get_class_fullname(self)}' was not yet set.") + if hasattr(self, "root_dir") and self.root_dir == constants.DUMMY_NONEXISTING_PATH: # type: ignore + raise SerializationError(f"root_dir attribute for '{get_class_fullname(self)}' was not yet set.") + + if Path(output_path).is_dir(): # type: ignore + raise SerializationError("output path for json cannot be directory.") + + # false positive MyPy warning, cannot be None + with Path(output_path).open("w") as handle: # type: ignore + json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False) + + @classmethod + def from_json(cls: type[T], input_path: str | Path) -> T: + input_path = Path(input_path) + with input_path.open("r") as handle: + obj = json.load(handle, cls=CustomJSONDecoder) + return obj + + +# Decorator for serialization +def serialize(func: Callable): + @wraps(func) + def inner_func(*args, **kwargs): + if not args or not issubclass(type(args[0]), ComplexSerializableType): + raise ValueError( + "@serialize decorator is to be used only on instance methods of ComplexSerializableType child classes." + ) + + if hasattr(args[0], "_root_dir") and args[0]._root_dir == constants.DUMMY_NONEXISTING_PATH: + raise SerializationError( + "The invoked method requires dataset serialization. Cannot serialize without root_dir set. You can set it with obj.root_dir = ..." + ) + + update_json = kwargs.pop("update_json", True) + result = func(*args, **kwargs) + if update_json: + args[0].to_json() + return result + + return inner_func + + +def get_class_fullname(obj: Any) -> str: + if isinstance(obj, type): + klass = obj + else: + klass = obj.__class__ + module = klass.__module__ + if module == "builtins": + return klass.__qualname__ + return module + "." + klass.__qualname__ + + +class CustomJSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, ComplexSerializableType): + return {**{"_type": get_class_fullname(obj)}, **obj.to_dict()} + if isinstance(obj, dict): + return obj + if isinstance(obj, set): + return {"_type": "Set", "elements": sorted(list(obj))} + if isinstance(obj, frozenset): + return sorted(list(obj)) + if isinstance(obj, date): + return str(obj) + if isinstance(obj, Path): + return str(obj) + return super().default(obj) + + +class CustomJSONDecoder(json.JSONDecoder): + """ + Custom JSONDecoder. Any complex object that should be de-serializable must inherit directly from class + ComplexSerializableType (nested inheritance does not currently work (because x.__subclassess__() prints only direct + subclasses. Any such class must implement methods to_dict() and from_dict(). These are used to drive serialization. + """ + + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + self.serializable_complex_types = {get_class_fullname(x): x for x in ComplexSerializableType.__subclasses__()} + + def object_hook(self, obj): + if "_type" in obj and obj["_type"] == "Set": + return set(obj["elements"]) + if "_type" in obj and obj["_type"] in self.serializable_complex_types.keys(): + complex_type = obj.pop("_type") + return self.serializable_complex_types[complex_type].from_dict(obj) + elif "_type" in obj: + raise SerializationError(f"JSONDecoder doesn't know how to handle {obj}") + + return obj diff --git a/src/sec_certs/serialization/pandas.py b/src/sec_certs/serialization/pandas.py new file mode 100644 index 00000000..dc0aae18 --- /dev/null +++ b/src/sec_certs/serialization/pandas.py @@ -0,0 +1,16 @@ +from abc import ABC, abstractmethod + + +class PandasSerializableType(ABC): + def __init__(self, *args, **kwargs): + pass + + @property + @abstractmethod + def pandas_tuple(self): + raise NotImplementedError("Not meant to be implemented") + + @property + @abstractmethod + def pandas_columns(self): + raise NotImplementedError("Not meant to be implemented") diff --git a/src/sec_certs/utils/__init__.py b/src/sec_certs/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/sec_certs/utils/extract.py b/src/sec_certs/utils/extract.py new file mode 100644 index 00000000..09460fb7 --- /dev/null +++ b/src/sec_certs/utils/extract.py @@ -0,0 +1,817 @@ +from __future__ import annotations + +import logging +import os +import re +from collections import Counter +from enum import Enum +from pathlib import Path +from typing import Any, Iterator + +import numpy as np + +from sec_certs import constants as constants +from sec_certs.cert_rules import REGEXEC_SEP, cc_rules +from sec_certs.constants import FILE_ERRORS_STRATEGY, LINE_SEPARATOR, MAX_ALLOWED_MATCH_LENGTH + +logger = logging.getLogger(__name__) + + +def search_only_headers_anssi(filepath: Path): # noqa: C901 + # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! + class HEADER_TYPE(Enum): + HEADER_FULL = 1 + HEADER_MISSING_CERT_ITEM_VERSION = 2 + HEADER_MISSING_PROTECTION_PROFILES = 3 + HEADER_DUPLICITIES = 4 + + rules_certificate_preface = [ + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\\(s\\)(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)Product name \\(reference / version\\)(.+)TOE name \\(reference / version\\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements", + ), + ( + HEADER_TYPE.HEADER_FULL, + "Certification report reference(.+)TOE name(.+)Product's reference/ version(.+)TOE's reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements", + ), + # corrupted text (duplicities) + ( + HEADER_TYPE.HEADER_DUPLICITIES, + "Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles", + ), + # rules without product version + ( + HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, + "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, + "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ( + HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, + "Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + # rules without protection profile + ( + HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES, + "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables", + ), + ] + + # statistics about rules success rate + num_rules_hits = {} + for rule in rules_certificate_preface: + num_rules_hits[rule[1]] = 0 + + items_found = {} # type: ignore # noqa + + try: + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath) + + # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs + pos = whole_text.find(" ") + if pos != -1: + pos = whole_text.find(" ", pos) + if pos != -1: + whole_text = whole_text[pos:] + + no_match_yet = True + other_rule_already_match = False + rule_index = -1 + for rule in rules_certificate_preface: + rule_index += 1 + rule_and_sep = rule[1] + REGEXEC_SEP + + for m in re.finditer(rule_and_sep, whole_text): + if no_match_yet: + items_found[constants.TAG_HEADER_MATCH_RULES] = [] + no_match_yet = False + + # insert rule if at least one match for it was found + if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]: + items_found[constants.TAG_HEADER_MATCH_RULES].append(rule[1]) + + if not other_rule_already_match: + other_rule_already_match = True + else: + logger.warning(f"WARNING: multiple rules are matching same certification document: {filepath}") + + num_rules_hits[rule[1]] += 1 # add hit to this rule + match_groups = m.groups() + index_next_item = 0 + items_found[constants.TAG_CERT_ID] = normalize_match_string(match_groups[index_next_item]) + index_next_item += 1 + + items_found[constants.TAG_CERT_ITEM] = normalize_match_string(match_groups[index_next_item]) + index_next_item += 1 + + if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION: + items_found[constants.TAG_CERT_ITEM_VERSION] = "" + else: + items_found[constants.TAG_CERT_ITEM_VERSION] = normalize_match_string(match_groups[index_next_item]) + index_next_item += 1 + + if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES: + items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = "" + else: + items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( + match_groups[index_next_item] + ) + index_next_item += 1 + + items_found[constants.TAG_CC_VERSION] = normalize_match_string(match_groups[index_next_item]) + index_next_item += 1 + + items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(match_groups[index_next_item]) + index_next_item += 1 + + items_found[constants.TAG_DEVELOPER] = normalize_match_string(match_groups[index_next_item]) + index_next_item += 1 + + items_found[constants.TAG_CERT_LAB] = normalize_match_string(match_groups[index_next_item]) + index_next_item += 1 + except Exception as e: + relative_filepath = "/".join(str(filepath).split("/")[-4:]) + error_msg = f"Failed to parse ANSSI frontpage headers from {relative_filepath}; {e}" + logger.error(error_msg) + return error_msg, None + + # if True: + # print('# hits for rule') + # sorted_rules = sorted(num_rules_hits.items(), + # key=operator.itemgetter(1), reverse=True) + # used_rules = [] + # for rule in sorted_rules: + # print('{:4d} : {}'.format(rule[1], rule[0])) + # if rule[1] > 0: + # used_rules.append(rule[0]) + + return constants.RETURNCODE_OK, items_found + + +def search_only_headers_bsi(filepath: Path): # noqa: C901 + # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! + LINE_SEPARATOR_STRICT = " " + NUM_LINES_TO_INVESTIGATE = 15 + rules_certificate_preface = [ + "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)", + "(BSI-DSZ-CC-.+?) zu (.+?) der (.*)", + ] + + items_found = {} # type: ignore # noqa + no_match_yet = True + + try: + # Process front page with info: cert_id, certified_item and developer + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( + filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT + ) + + for rule in rules_certificate_preface: + rule_and_sep = rule + REGEXEC_SEP + + for m in re.finditer(rule_and_sep, whole_text): + if no_match_yet: + items_found[constants.TAG_HEADER_MATCH_RULES] = [] + no_match_yet = False + + # insert rule if at least one match for it was found + if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]: + items_found[constants.TAG_HEADER_MATCH_RULES].append(rule) + + match_groups = m.groups() + cert_id = match_groups[0] + certified_item = match_groups[1] + developer = match_groups[2] + + FROM_KEYWORD_LIST = [" from ", " der "] + for from_keyword in FROM_KEYWORD_LIST: + from_keyword_len = len(from_keyword) + if certified_item.find(from_keyword) != -1: + logger.warning( + f"string {from_keyword} detected in certified item - shall not be here, fixing..." + ) + certified_item_first = certified_item[: certified_item.find(from_keyword)] + developer = certified_item[certified_item.find(from_keyword) + from_keyword_len :] + certified_item = certified_item_first + continue + + end_pos = developer.find("\f-") + if end_pos == -1: + end_pos = developer.find("\fBSI") + if end_pos == -1: + end_pos = developer.find("Bundesamt") + if end_pos != -1: + developer = developer[:end_pos] + + items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) + items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item) + items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer) + items_found[constants.TAG_CERT_LAB] = "BSI" + + # Process page with more detailed sample info + # PP Conformance, Functionality, Assurance + rules_certificate_third = ["PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified"] + + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath) + + for rule in rules_certificate_third: + rule_and_sep = rule + REGEXEC_SEP + + for m in re.finditer(rule_and_sep, whole_text): + # check if previous rules had at least one match + if constants.TAG_CERT_ID not in items_found.keys(): + logger.error(f"ERROR: front page not found for file: {filepath}") + + match_groups = m.groups() + ref_protection_profiles = match_groups[0] + cc_version = match_groups[1] + cc_security_level = match_groups[2] + + items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( + ref_protection_profiles + ) + items_found[constants.TAG_CC_VERSION] = normalize_match_string(cc_version) + items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(cc_security_level) + + # print('\n*** Certificates without detected preface:') + # for file_name in files_without_match: + # print('No hits for {}'.format(file_name)) + # print('Total no hits files: {}'.format(len(files_without_match))) + # print('\n**********************************') + except Exception as e: + relative_filepath = "/".join(str(filepath).split("/")[-4:]) + error_msg = f"Failed to parse BSI headers from frontpage: {relative_filepath}; {e}" + logger.error(error_msg) + return error_msg, None + + return constants.RETURNCODE_OK, items_found + + +def search_only_headers_nscib(filepath: Path): # noqa: C901 + # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! + LINE_SEPARATOR_STRICT = " " + NUM_LINES_TO_INVESTIGATE = 60 + items_found: dict[str, str] = {} + + try: + # Process front page with info: cert_id, certified_item and developer + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( + filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT + ) + + certified_item = "" + developer = "" + cert_lab = "" + cert_id = "" + + lines = whole_text_with_newlines.splitlines() + no_match_yet = True + item_offset = -1 + + for line_index in range(0, len(lines)): + line = lines[line_index] + + if "Certification Report" in line: + item_offset = line_index + 1 + if "Assurance Continuity Maintenance Report" in line: + item_offset = line_index + 1 + + SPONSORDEVELOPER_STR = "Sponsor and developer:" + + if SPONSORDEVELOPER_STR in line: + if no_match_yet: + items_found = {} + no_match_yet = False + + # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report' + certified_item = "" + for name_index in range(item_offset, line_index): + certified_item += lines[name_index] + " " + developer = line[line.find(SPONSORDEVELOPER_STR) + len(SPONSORDEVELOPER_STR) :] + + SPONSOR_STR = "Sponsor:" + + if SPONSOR_STR in line: + if no_match_yet: + items_found = {} + no_match_yet = False + + # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report' + certified_item = "" + for name_index in range(item_offset, line_index): + certified_item += lines[name_index] + " " + + DEVELOPER_STR = "Developer:" + if DEVELOPER_STR in line: + developer = line[line.find(DEVELOPER_STR) + len(DEVELOPER_STR) :] + + CERTLAB_STR = "Evaluation facility:" + if CERTLAB_STR in line: + cert_lab = line[line.find(CERTLAB_STR) + len(CERTLAB_STR) :] + + REPORTNUM_STR = "Report number:" + if REPORTNUM_STR in line: + cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :] + + if not no_match_yet: + items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) + items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item) + items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer) + items_found[constants.TAG_CERT_LAB] = cert_lab + + except Exception as e: + error_msg = f"Failed to parse NSCIB headers from frontpage: {filepath}; {e}" + logger.error(error_msg) + return error_msg, None + + return constants.RETURNCODE_OK, items_found + + +def search_only_headers_niap(filepath: Path): + # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! + LINE_SEPARATOR_STRICT = " " + NUM_LINES_TO_INVESTIGATE = 15 + items_found: dict[str, str] = {} + + try: + # Process front page with info: cert_id, certified_item and developer + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( + filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT + ) + + certified_item = "" + cert_id = "" + + lines = whole_text_with_newlines.splitlines() + no_match_yet = True + item_offset = -1 + + for line_index in range(0, len(lines)): + line = lines[line_index] + + if "Validation Report" in line: + item_offset = line_index + 1 + + REPORTNUM_STR = "Report Number:" + if REPORTNUM_STR in line: + if no_match_yet: + items_found = {} + no_match_yet = False + + # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report' + certified_item = "" + for name_index in range(item_offset, line_index): + certified_item += lines[name_index] + " " + cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :] + break + + if not no_match_yet: + items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) + items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item) + items_found[constants.TAG_CERT_LAB] = "US NIAP" + + except Exception as e: + error_msg = f"Failed to parse NIAP headers from frontpage: {filepath}; {e}" + logger.error(error_msg) + return error_msg, None + + return constants.RETURNCODE_OK, items_found + + +def search_only_headers_canada(filepath: Path): # noqa: C901 + # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!! + LINE_SEPARATOR_STRICT = " " + NUM_LINES_TO_INVESTIGATE = 20 + items_found: dict[str, str] = {} + try: + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file( + filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT + ) + + cert_id = "" + + lines = whole_text_with_newlines.splitlines() + no_match_yet = True + for line_index in range(0, len(lines)): + line = lines[line_index] + if "Government of Canada, Communications Security Establishment" in line: + REPORTNUM_STR1 = "Evaluation number:" + REPORTNUM_STR2 = "Document number:" + matched_number_str = "" + line_certid = lines[line_index + 1] + if line_certid.startswith(REPORTNUM_STR1): + matched_number_str = REPORTNUM_STR1 + if line_certid.startswith(REPORTNUM_STR2): + matched_number_str = REPORTNUM_STR2 + if matched_number_str != "": + if no_match_yet: + items_found = {} + no_match_yet = False + + cert_id = line_certid[line_certid.find(matched_number_str) + len(matched_number_str) :] + break + + if ( + "Government of Canada. This document is the property of the Government of Canada. It shall not be altered," + in line + ): + REPORTNUM_STR = "Evaluation number:" + for offset in range(1, 20): + line_certid = lines[line_index + offset] + if "UNCLASSIFIED" in line_certid: + if no_match_yet: + items_found = {} + no_match_yet = False + line_certid = lines[line_index + offset - 4] + cert_id = line_certid[line_certid.find(REPORTNUM_STR) + len(REPORTNUM_STR) :] + break + if not no_match_yet: + break + + if ( + "UNCLASSIFIED / NON CLASSIFIÉ" in line + and "COMMON CRITERIA CERTIFICATION REPORT" in lines[line_index + 2] + ): + line_certid = lines[line_index + 1] + if no_match_yet: + items_found = {} + no_match_yet = False + cert_id = line_certid + break + + if not no_match_yet and cert_id: + items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id) + items_found[constants.TAG_CERT_LAB] = "CANADA" + + except Exception as e: + error_msg = f"Failed to parse Canada headers from frontpage: {filepath}; {e}" + logger.error(error_msg) + return error_msg, None + + return constants.RETURNCODE_OK, items_found + + +def search_files(folder: str | Path) -> Iterator[str]: + for root, _, files in os.walk(str(folder)): + yield from [os.path.join(root, x) for x in files] + + +def flatten_matches(dct: dict) -> dict: + """ + Function to flatten dictionary of matches. + + Turns + ``` + {"a": {"cc": 3}, "b": {}, "d": {"dd": 4, "cc": 2}} + ``` + into + ``` + {"cc": 5, "dd": 4} + ``` + + :param dct: Dictionary to flatten + :return: Flattened dictionary + """ + result: Counter[Any] = Counter() + for key, value in dct.items(): + if isinstance(value, dict): + result.update(flatten_matches(value)) + else: + result[key] = value + return dict(result) + + +def prune_matches(dct: dict) -> dict: + """ + Prune a dictionary of matches. + + Turns + ``` + {"a": {"cc": 3}, "b": {"aa": {}, "bb": {}}, "d": {"dd": 4, "cc": 2}} + ``` + into + ``` + {"a": {"cc": 3}, "b": {}, "d": {"dd": 4, "cc": 2}} + ``` + + :param dct: The dictionary of matches. + :return: The pruned dictionary. + """ + + def walk(obj, depth): + if isinstance(obj, dict): + if not obj: + return None + res = {} + for k, v in obj.items(): + r = walk(v, depth + 1) + if r is not None: + res[k] = r + return res if res or depth == 1 else None + else: + return obj + + return walk(dct, 0) + + +def extract_keywords(filepath: Path, search_rules) -> dict[str, dict[str, int]] | None: + """ + Extract keywords from filepath using the search rules. + + :param filepath: + :param search_rules: + :return: + """ + + try: + whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath, -1, LINE_SEPARATOR) + + def extract(rules): + if isinstance(rules, dict): + return {k: extract(v) for k, v in rules.items()} + elif isinstance(rules, list): + matches = [extract(rule) for rule in rules] + c = Counter() + for match_list in matches: + c += Counter(match_list) + return dict(c) + elif isinstance(rules, re.Pattern): + rule = rules + matches = [] + for match in rule.finditer(whole_text): + match = match.group("match") + match = normalize_match_string(match) + + match_len = len(match) + if match_len > MAX_ALLOWED_MATCH_LENGTH: + logger.warning(f"Excessive match with length of {match_len} detected for rule {rule.pattern}") + matches.append(match) + return matches + + result = extract(search_rules) + return prune_matches(result) + except Exception as e: + relative_filepath = "/".join(str(filepath).split("/")[-4:]) + error_msg = f"Failed to parse keywords from: {relative_filepath}; {e}" + logger.error(error_msg) + return None + + +def normalize_match_string(match: str) -> str: + match = match.strip().strip("[];.”\"':)(,").rstrip(os.sep).replace(" ", " ") + return "".join(filter(str.isprintable, match)) + + +def load_text_file( + file_name: str | Path, limit_max_lines: int = -1, line_separator: str = LINE_SEPARATOR +) -> tuple[str, str, bool]: + """ + Load the text contents of a file at `file_name`, upto `limit_max_lines` of lines, replace + newlines in the text with `line_separator`. + + :param file_name: The file_name to load. + :param limit_max_lines: The limit on number of lines to return. + :param line_separator: The string to replace newlines with. + :return: A tuple of three elements (the text with replaced newlines, the text and a boolean whether a unicode + decoding error happened). + """ + lines = [] + was_unicode_decode_error = False + with Path(file_name).open("r", errors=FILE_ERRORS_STRATEGY) as f: + try: + lines = f.readlines() + except UnicodeDecodeError: + was_unicode_decode_error = True + logger.warning("UnicodeDecodeError, opening as utf8") + + if was_unicode_decode_error: + with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2: + # coding failure, try line by line + line = " " + while line: + try: + line = f2.readline() + lines.append(line) + except UnicodeDecodeError: + # ignore error + continue + + whole_text = "" + whole_text_with_newlines = "" + lines_included = 0 + for line in lines: + if limit_max_lines != -1 and lines_included >= limit_max_lines: + break + + whole_text_with_newlines += line + line = line.replace("\n", "") + whole_text += line + whole_text += line_separator + lines_included += 1 + + return whole_text, whole_text_with_newlines, was_unicode_decode_error + + +def load_cert_html_file(file_name: str) -> str: + with open(file_name, errors=FILE_ERRORS_STRATEGY) as f: + try: + return f.read() + except UnicodeDecodeError: + logger.warning("UnicodeDecodeError, opening as utf8") + + with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2: + try: + return f2.read() + except UnicodeDecodeError: + logger.error(f"Failed to read file {file_name}") + return "" + + +def rules_get_subset(desired_path: str) -> dict: + """ + Recursively applies cc_certs.get(key) on tokens from desired_path, + returns the keys of the inner-most layer. + """ + dct = cc_rules + for token in desired_path.split("."): + dct = dct[token] + return dct + + +def extract_key_paths(dct: dict, current_path: str) -> list[str]: + """ + Given subset of cc_rules dictionary, will compute full paths to all leafs + in the dictionaries, s.t. the final value of each path is a list of regex + matches in the keywords dictionary. + """ + paths = [] + for key in dct: + if isinstance(dct[key], dict): + paths.extend(extract_key_paths(dct[key], current_path + "." + key)) + elif isinstance(dct[key], list): + paths.append(current_path + "." + key) + return paths + + +def get_sum_of_values_from_dict_path(dct: dict | None, path: str, default: float = np.nan) -> float: + """ + Given dictionary and path, will compute sum of occurences of values in the inner-most layer + of that path. If the key is missing from dict, return default value. + """ + if not dct: + return np.nan + + res = dct + + try: + for token in path.split("."): + res = res[token] + except KeyError: + return default + + return sum(res.values()) + + +def get_sums_for_rules_subset(dct: dict | None, path: str) -> dict[str, float]: + """ + Given path to search in cc_rules (e.g., "symmetric_crypto"), + will get the finest resolution and count occurences of the keys in the + examined dictionary. + """ + cc_rules_subset_to_search = rules_get_subset(path) + paths_to_search = extract_key_paths(cc_rules_subset_to_search, path) + return {x: get_sum_of_values_from_dict_path(dct, x, np.nan) for x in paths_to_search} diff --git a/src/sec_certs/utils/helpers.py b/src/sec_certs/utils/helpers.py new file mode 100644 index 00000000..302f4e6a --- /dev/null +++ b/src/sec_certs/utils/helpers.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import hashlib +import logging +import re +import time +from contextlib import nullcontext +from datetime import datetime +from functools import partial +from pathlib import Path +from typing import Any, Collection + +import numpy as np +import pkgconfig +import requests + +import sec_certs.constants as constants +from sec_certs.config.configuration import config +from sec_certs.utils import parallel_processing +from sec_certs.utils.tqdm import tqdm + +logger = logging.getLogger(__name__) + + +def download_file( + url: str, output: Path, delay: float = 0, show_progress_bar: bool = False, progress_bar_desc: str | None = None +) -> str | int: + try: + time.sleep(delay) + # See https://github.com/psf/requests/issues/3953 for header justification + r = requests.get( + url, allow_redirects=True, timeout=constants.REQUEST_TIMEOUT, stream=True, headers={"Accept-Encoding": None} # type: ignore + ) + ctx: Any + if show_progress_bar: + ctx = partial( + tqdm, + total=int(r.headers.get("content-length", 0)), + unit="B", + unit_scale=True, + unit_divisor=1024, + desc=progress_bar_desc, + ) + else: + ctx = nullcontext + + if r.status_code == requests.codes.ok: + with ctx() as pbar: + with output.open("wb") as f: + for data in r.iter_content(1024): + f.write(data) + if show_progress_bar: + pbar.update(len(data)) + + return r.status_code + except requests.exceptions.Timeout: + return requests.codes.timeout + except Exception as e: + logger.error(f"Failed to download from {url}; {e}") + return constants.RETURNCODE_NOK + return constants.RETURNCODE_NOK + + +def download_parallel( + urls: Collection[str], paths: Collection[Path], progress_bar_desc: str | None = None +) -> list[int]: + exit_codes = parallel_processing.process_parallel( + download_file, list(zip(urls, paths)), config.n_threads, unpack=True, progress_bar_desc=progress_bar_desc + ) + n_successful = len([e for e in exit_codes if e == requests.codes.ok]) + logger.info(f"Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.") + + for url, e in zip(urls, exit_codes): + if e != requests.codes.ok: + logger.error(f"Failed to download {url}, exit code: {e}") + + return exit_codes + + +def fips_dgst(cert_id: int | str) -> str: + return get_first_16_bytes_sha256(str(cert_id)) + + +def get_first_16_bytes_sha256(string: str) -> str: + return hashlib.sha256(string.encode("utf-8")).hexdigest()[:16] + + +def get_sha256_filepath(filepath: str | Path) -> str: + hash_sha256 = hashlib.sha256() + with Path(filepath).open("rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_sha256.update(chunk) + return hash_sha256.hexdigest() + + +def to_utc(timestamp: datetime) -> datetime: + offset = timestamp.utcoffset() + if offset is None: + return timestamp + timestamp -= offset + timestamp = timestamp.replace(tzinfo=None) + return timestamp + + +def is_in_dict(target_dict: dict, path: str) -> bool: + current_level = target_dict + for item in path: + if item not in current_level: + return False + else: + current_level = current_level[item] + return True + + +def compute_heuristics_version(cert_name: str) -> set[str]: + """ + Will extract possible versions from the name of sample + """ + at_least_something = r"(\b(\d)+\b)" + just_numbers = r"(\d{1,5})(\.\d{1,5})" + + without_version = r"(" + just_numbers + r"+)" + long_version = r"(" + r"(\bversion)\s*" + just_numbers + r"+)" + short_version = r"(" + r"\bv\s*" + just_numbers + r"+)" + full_regex_string = r"|".join([without_version, short_version, long_version]) + normalizer = r"(\d+\.*)+" + + matched_strings = [max(x, key=len) for x in re.findall(full_regex_string, cert_name, re.IGNORECASE)] + if not matched_strings: + matched_strings = [max(x, key=len) for x in re.findall(at_least_something, cert_name, re.IGNORECASE)] + # Only keep the first occurrence but keep order. + matches = [] + for match in matched_strings: + if match not in matches: + matches.append(match) + # identified_versions = list(set([max(x, key=len) for x in re.findall(VERSION_PATTERN, cert_name, re.IGNORECASE | re.VERBOSE)])) + # return identified_versions if identified_versions else ['-'] + + if not matches: + return {constants.CPE_VERSION_NA} + + matched = [re.search(normalizer, x) for x in matches] + return {x.group() for x in matched if x is not None} + + +def tokenize_dataset(dset: list[str], keywords: set[str]) -> np.ndarray: + return np.array([tokenize(x, keywords) for x in dset]) + + +def tokenize(string: str, keywords: set[str]) -> str: + return " ".join([x for x in string.split() if x.lower() in keywords]) + + +def normalize_fips_vendor(string: str) -> str: + """ + "Normalizes" FIPS vendor. Precisely: + - Removes some punctuation and non-alphanumerical symbols + - Returns only first 5 tokens + # TODO: The rationale of the steps outlined above should be investigatated + """ + return " ".join( + string.replace("(R)", "").replace(",", "").replace("®", "").replace("-", " ").replace("+", " ").split()[:4] + ) + + +# Credit: https://stackoverflow.com/questions/18092354/ +def split_unescape(s: str, delim: str, escape: str = "\\", unescape: bool = True) -> list[str]: + """ + >>> split_unescape('foo,bar', ',') + ['foo', 'bar'] + >>> split_unescape('foo$,bar', ',', '$') + ['foo,bar'] + >>> split_unescape('foo$$,bar', ',', '$', unescape=True) + ['foo$', 'bar'] + >>> split_unescape('foo$$,bar', ',', '$', unescape=False) + ['foo$$', 'bar'] + >>> split_unescape('foo$', ',', '$', unescape=True) + ['foo$'] + """ + ret = [] + current = [] + itr = iter(s) + for ch in itr: + if ch == escape: + try: + # skip the next character; it has been escaped! + if not unescape: + current.append(escape) + current.append(next(itr)) + except StopIteration: + if unescape: + current.append(escape) + elif ch == delim: + # split! (add current to the list and reset it) + ret.append("".join(current)) + current = [] + else: + current.append(ch) + ret.append("".join(current)) + return ret + + +def warn_if_missing_poppler() -> None: + """ + Warns user if he misses a poppler dependency + """ + try: + if not pkgconfig.installed("poppler-cpp", ">=0.30"): + logger.warning( + "Attempting to run pipeline with pdf->txt conversion, but poppler-cpp dependency was not found." + ) + except OSError: + logger.warning("Attempting to find poppler-cpp, but pkg-config was not found.") + + +def warn_if_missing_tesseract() -> None: + """ + Warns user if he misses a tesseract dependency + """ + try: + if not pkgconfig.installed("tesseract", ">=5.0.0"): + logger.warning( + "Attempting to run pipeline with pdf->txt conversion, that requires tesseract, but tesseract was not found." + ) + except OSError: + logger.warning("Attempting to find tesseract, but pkg-config was not found.") + + +def choose_lowest_eal(eals: set[str] | None) -> str | None: + """ + Given a set of EAL strings, chooses the lowest one. + """ + if not eals: + return None + + matches = [(re.search(r"\d+", x)) for x in eals] + min_number = min([int(x.group()) for x in matches if x]) + candidates = [x for x in eals if str(min_number) in x] + return "EAL" + str(min_number) if len(candidates) == 2 else candidates[0] diff --git a/src/sec_certs/utils/pandas.py b/src/sec_certs/utils/pandas.py new file mode 100644 index 00000000..97068e77 --- /dev/null +++ b/src/sec_certs/utils/pandas.py @@ -0,0 +1,542 @@ +from __future__ import annotations + +import copy +import functools +import logging +import tempfile +import xml.etree.ElementTree as ET +import zipfile +from dataclasses import dataclass +from pathlib import Path +from shutil import copyfile +from typing import Any, Final + +import numpy as np +import pandas as pd +from matplotlib import pyplot as plt +from scipy import stats +from tqdm.notebook import tqdm + +from sec_certs.dataset.cve import CVEDataset +from sec_certs.sample.sar import SAR +from sec_certs.utils import helpers + +logger = logging.getLogger(__name__) + + +@dataclass(eq=True, frozen=True) +class SecondarySFPCluster: + name: str + children: frozenset[int] + + @classmethod + def from_xml_id(cls, xml_categories: list[ET.Element], cwe_id: int): + cat = cls.find_correct_category(xml_categories, cwe_id) + name = cat.attrib["Name"] + members = cat.find("{http://cwe.mitre.org/cwe-6}Relationships") + + assert members is not None + member_ids = frozenset( + int(x.attrib["CWE_ID"]) for x in members if x.tag == "{http://cwe.mitre.org/cwe-6}Has_Member" + ) + return cls(name, member_ids) + + @staticmethod + def find_correct_category(xml_categories: list[ET.Element], cwe_id: int) -> ET.Element: + for cat in xml_categories: + if cat.attrib["ID"] == str(cwe_id): + return cat + raise ValueError(f"Category with ID {cwe_id} found.") + + +@dataclass(eq=True, frozen=True) +class PrimarySFPCluster: + name: str + secondary_clusters: frozenset[SecondarySFPCluster] + cwe_ids: frozenset[int] + + @classmethod + def from_xml(cls, xml_categories: list[ET.Element], primary_cluster_element: ET.Element): + name = primary_cluster_element.attrib["Name"].split("SFP Primary Cluster: ")[1] + members = primary_cluster_element.find("{http://cwe.mitre.org/cwe-6}Relationships") + + assert members is not None + member_ids = {int(x.attrib["CWE_ID"]) for x in members if x.tag == "{http://cwe.mitre.org/cwe-6}Has_Member"} + + secondary_clusters = [] + cwe_ids = [] + for member_id in member_ids: + try: + secondary_clusters.append(SecondarySFPCluster.from_xml_id(xml_categories, member_id)) + except ValueError: + cwe_ids.append(member_id) + + return cls(name, frozenset(secondary_clusters), frozenset(cwe_ids)) + + +class SFPModel: + URL: Final[str] = "https://cwe.mitre.org/data/xml/views/888.xml.zip" + XML_FILENAME: Final[str] = "888.xml" + XML_ZIP_NAME: Final[str] = "888.xml.zip" + + def __init__(self, primary_clusters: frozenset[PrimarySFPCluster]): + self.primary_clusters = primary_clusters + + @classmethod + def from_xml(cls, xml_filepath: str | Path): + tree = ET.parse(xml_filepath) + category_tag = tree.getroot().find("{http://cwe.mitre.org/cwe-6}Categories") + + assert category_tag is not None + categories = category_tag.findall("{http://cwe.mitre.org/cwe-6}Category") + + # The XML contains two weird primary clusters not specified in https://samate.nist.gov/BF/Enlightenment/SFP.html. + # After manual inspection, we skip those + primary_clusters = frozenset( + PrimarySFPCluster.from_xml(categories, x) + for x in categories + if ( + "SFP Primary Cluster" in x.attrib["Name"] + and x.attrib["Name"] != "SFP Primary Cluster: Failure to Release Memory" + and x.attrib["Name"] != "SFP Primary Cluster: Faulty Resource Release" + ) + ) + + return cls(primary_clusters) + + @classmethod + def from_web(cls): + with tempfile.TemporaryDirectory() as tmp_dir: + xml_zip_path = Path(tmp_dir) / cls.XML_ZIP_NAME + helpers.download_file(cls.URL, xml_zip_path) + + with zipfile.ZipFile(xml_zip_path, "r") as zip_handle: + zip_handle.extractall(tmp_dir) + + return cls.from_xml(Path(tmp_dir) / cls.XML_FILENAME) + + def search_cwe(self, cwe_id: int) -> tuple[str | None, str | None]: + for primary in self.primary_clusters: + for secondary in primary.secondary_clusters: + if cwe_id in secondary.children: + return primary.name, secondary.name + if cwe_id in primary.cwe_ids: + return primary.name, None + return None, None + + +def discover_sar_families(ser: pd.Series) -> list[str]: + """ + Returns a list of all SAR families that occur in the pandas Series, where each entry is a set of SAR objects. + """ + sars = ser.tolist() + families = set() + for cert in sars: + families |= {x.family for x in cert} if not pd.isnull(cert) else set() + return list(families) + + +def get_sar_level_from_set(sars: set[SAR], sar_family: str) -> int | None: + """ + Given a set of SARs and a family name, will return level of the seeked SAR from the set. + """ + family_sars_dict = {x.family: x for x in sars} if (sars and not pd.isnull(sars)) else dict() + if sar_family not in family_sars_dict.keys(): + return None + return family_sars_dict[sar_family].level + + +def compute_cve_correlations( + df: pd.DataFrame, + exclude_vuln_free_certs: bool = False, + sar_families: list[str] | None = None, + output_path: str | Path | None = None, + filter_nans: bool = True, +) -> pd.DataFrame: + """ + Computes correlations of EAL and different SARs and two columns: (n_cves, worst_cve_score, avg_cve_score). Few assumptions about the passed dataframe: + - EAL column must be categorical data type + - SAR column must be a set of SARs + - `n_cves` and `worst_cve_score`, `avg_cve_score` columns must be present in the dataframe + Possibly, it can filter columns will both values NaN (due to division by zero or super low supports.) + To choose correct minimal support is tricky, this is because SAR levels often having huge support, but being imbalanced themselves heavily in the favor + of a single value that is rarely modified. We recommend choosing 100 and discarding any row where some column would result into NaN + """ + df_sar = df.loc[:, ["eal", "extracted_sars", "worst_cve_score", "avg_cve_score", "n_cves", "category"]] + df_sar = df_sar.loc[df_sar.category != "ICs, Smart Cards and Smart Card-Related Devices and Systems"] + + if exclude_vuln_free_certs: + df_sar = df_sar.loc[df_sar.n_cves > 0] + + families = sar_families if sar_families else discover_sar_families(df_sar.extracted_sars) + + spearmanr = functools.partial(stats.spearmanr, nan_policy="omit", alternative="less") + + df_sar.eal = df_sar.eal.cat.codes + df_sar.eal = df_sar.eal.map(lambda x: np.NaN if x == -1 else x) + + n_cves_eal_corr, n_cves_eal_pvalue = spearmanr(df_sar.eal, df_sar.n_cves) + n_cves_corrs = [n_cves_eal_corr] + n_cves_pvalues = [n_cves_eal_pvalue] + + worst_cve_eal_corr, worst_cve_eal_pvalue = spearmanr(df_sar.eal, df_sar.worst_cve_score) + worst_cve_corrs = [worst_cve_eal_corr] + worst_cve_pvalues = [worst_cve_eal_pvalue] + + avg_cve_eal_corr, avg_cve_eal_pvalue = spearmanr(df_sar.eal, df_sar.avg_cve_score) + avg_cve_corrs = [avg_cve_eal_corr] + avg_cve_pvalues = [avg_cve_eal_pvalue] + + supports = [df_sar.loc[~df_sar["eal"].isnull()].shape[0]] + + for family in tqdm(families): + df_sar[family] = df_sar.extracted_sars.map(lambda x: get_sar_level_from_set(x, family)) + + n_cves_corr, n_cves_pvalue = spearmanr(df_sar[family], df_sar.n_cves) + n_cves_corrs.append(n_cves_corr) + n_cves_pvalues.append(n_cves_pvalue) + + worst_cve_corr, worst_cve_pvalue = spearmanr(df_sar[family], df_sar.worst_cve_score) + worst_cve_corrs.append(worst_cve_corr) + worst_cve_pvalues.append(worst_cve_pvalue) + + avg_cve_corr, avg_cve_pvalue = spearmanr(df_sar[family], df_sar.avg_cve_score) + avg_cve_corrs.append(avg_cve_corr) + avg_cve_pvalues.append(avg_cve_pvalue) + + supports.append(df_sar.loc[~df_sar[family].isnull()].shape[0]) + + df_sar = df_sar.copy() + + tuples = list( + zip(n_cves_corrs, n_cves_pvalues, worst_cve_corrs, worst_cve_pvalues, avg_cve_corrs, avg_cve_pvalues, supports) + ) + dct = {family: correlations for family, correlations in zip(["eal"] + families, tuples)} + df_corr = pd.DataFrame.from_dict( + dct, + orient="index", + columns=[ + "n_cves_corr", + "n_cves_pvalue", + "worst_cve_score_corr", + "worst_cve_pvalue", + "avg_cve_score_corr", + "avg_cve_pvalue", + "support", + ], + ) + df_corr.style.set_caption("Correlations between EAL, SARs and CVEs") + df_corr = df_corr.sort_values(by="support", ascending=False) + + if filter_nans: + df_corr = df_corr.dropna(how="any", subset=["n_cves_corr", "worst_cve_score_corr", "avg_cve_score_corr"]) + + if output_path: + df_corr.to_csv(output_path) + + return df_corr + + +def find_earliest_maintenance_after_cve(row): + "Given dataframe row, will return first maintenance date succeeding first published CVE related to a certificate if exists, else np.nan" + maintenances_after_cve = [x for x in row["maintenance_dates"] if x > row["earliest_cve"]] + return min(maintenances_after_cve) if maintenances_after_cve else np.nan + + +def filter_to_cves_within_validity_period(cc_df: pd.DataFrame, cve_dset: CVEDataset) -> pd.DataFrame: + """ + Filters the column `related_cves` in `cc_df` DataFrame to CVEs that were published within validity period of the + studied certificate. + """ + + def filter_cves( + cve_dset: CVEDataset, cves: set[str], not_valid_before: pd.Timestamp, not_valid_after: pd.Timestamp + ) -> set[str] | float: + + # Mypy is complaining, but the Optional date is resolved at the beginning of the and condition + result: set[str] = { + x + for x in cves + if cve_dset[x].published_date + and not_valid_before < pd.Timestamp(cve_dset[x].published_date.date()) # type: ignore + and not_valid_after > pd.Timestamp(cve_dset[x].published_date.date()) # type: ignore + } + + return result if result else np.nan + + if ( + cc_df.loc[ + (cc_df.related_cves.notnull()) & ((cc_df.not_valid_before.isna()) | (cc_df.not_valid_after.isna())) + ].shape[0] + > 0 + ): + raise ValueError( + "Cannot filter CVEs on certificates that have NaNs in not_valid_after or not_valid_before fields." + ) + + cc_df["related_cves"] = cc_df.apply( + lambda row: filter_cves(cve_dset, row["related_cves"], row["not_valid_before"], row["not_valid_after"]) + if not pd.isna(row["related_cves"]) + else row["related_cves"], + axis=1, + ) + + return cc_df + + +def expand_df_with_cve_cols(df: pd.DataFrame, cve_dset: CVEDataset) -> pd.DataFrame: + df = df.copy() + + df["n_cves"] = df.related_cves.map(lambda x: len(x) if x is not np.nan else 0) + df["cve_published_dates"] = df.related_cves.map( + lambda x: [cve_dset[y].published_date.date() for y in x] if x is not np.nan else np.nan # type: ignore + ) + + df["earliest_cve"] = df.cve_published_dates.map(lambda x: min(x) if isinstance(x, list) else np.nan) + df["worst_cve_score"] = df.related_cves.map( + lambda x: max([cve_dset[cve].impact.base_score for cve in x]) if x is not np.nan else np.nan + ) + + """ + Note: Technically, CVE can have 0 base score. This happens when the CVE is discarded from the database. + This could skew the results. During May 2022 analysis, we encountered a single CVE with such score. + Therefore, we do not treat this case. + To properly treat this, the average should be taken across CVEs with >0 base_socre. + """ + df["avg_cve_score"] = df.related_cves.map( + lambda x: np.mean([cve_dset[cve].impact.base_score for cve in x]) if x is not np.nan else np.nan + ) + return df + + +def prepare_cwe_df( + cc_df: pd.DataFrame, cve_dset: CVEDataset, fine_grained: bool = False +) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + This function does the following: + 1. Filter CC DF to columns relevant for CWE examination (eal, related_cves, category) + 2. Parses CWE webpage of CWE categories and weaknesses, fetches CWE descriptions and names from there + 3. Explodes the CC DF so that each row corresponds to single CVE + 4. Joins CC DF with CWE DF obtained from CVEDataset + 5. Explodes resulting DF again so that each row corresponds to single CWE + + :param pd.DataFrame cc_df: DataFrame obtained from CCDataset, should be limited to rows with >0 vulnerabilities + :param CVEDataset cve_dset: CVEDataset instance to retrieve CWE data from + :param bool fine_grained: If se to True, CWEs won't be merged into weaknesses of higher abstraction + :return Tuple[pd.DataFrame, pd.DataFrame]: returns two dataframes: + - DF obtained from CC Dataset, fully exploded to CWEs + - DF obtained from CWE webpage, contains IDs, names, types, urls of all CWEs + """ + # Explode CVE_IDs and CWE_IDs so that we have right counts on duplicated CVEs. Measure how much data for analysis we have left. + vulns = cve_dset.to_pandas() + df_cwe_relevant = ( + cc_df[["eal", "related_cves", "category"]] + .explode(column="related_cves") + .rename(columns={"related_cves": "cve_id"}) + ) + df_cwe_relevant["cwe_ids"] = df_cwe_relevant.cve_id.map(lambda x: vulns.cwe_ids[x]) + df_cwe_relevant = ( + df_cwe_relevant.explode(column="cwe_ids") + .reset_index() + .rename(columns={"cwe_ids": "cwe_id", "index": "cert_dgst"}) + ) + + df_cwe_relevant.cwe_id = df_cwe_relevant.cwe_id.replace(r"NVD-CWE-*", np.nan, regex=True) + print( + f"Filtering {df_cwe_relevant.loc[df_cwe_relevant.cwe_id.isna(), 'cve_id'].nunique()} CVEs that have no CWE assigned. This affects {df_cwe_relevant.loc[df_cwe_relevant.cwe_id.isna(), 'cert_dgst'].nunique()} certificates" + ) + print( + f"Still left with analysis of {df_cwe_relevant.loc[~df_cwe_relevant.cwe_id.isna(), 'cve_id'].nunique()} CVEs in {df_cwe_relevant.loc[~df_cwe_relevant.cwe_id.isna(), 'cert_dgst'].nunique()} certificates." + ) + df_cwe_relevant = df_cwe_relevant.dropna() + + # Load CWE IDs and descriptions from CWE website + with tempfile.TemporaryDirectory() as tmp_dir: + xml_zip_path = Path(tmp_dir) / "cwec_latest.xml.zip" + helpers.download_file("https://cwe.mitre.org/data/xml/cwec_latest.xml.zip", xml_zip_path) + + with zipfile.ZipFile(xml_zip_path, "r") as zip_handle: + zip_handle.extractall(tmp_dir) + xml_filename = zip_handle.namelist()[0] + + root = ET.parse(Path(tmp_dir) / xml_filename).getroot() + + weaknesses = root.find("{http://cwe.mitre.org/cwe-6}Weaknesses") + categories = root.find("{http://cwe.mitre.org/cwe-6}Categories") + dct: dict[str, Any] = { + "cwe_id": [], + "cwe_name": [], + "cwe_description": [], + "type": [], + "child_of": [], + } + + assert weaknesses + for weakness in weaknesses: + assert weakness + description = weakness.find("{http://cwe.mitre.org/cwe-6}Description") + related_weaknesses = weakness.find("{http://cwe.mitre.org/cwe-6}Related_Weaknesses") + + dct["cwe_id"].append("CWE-" + weakness.attrib["ID"]) + dct["cwe_name"].append(weakness.attrib["Name"]) + dct["cwe_description"].append(description.text if description is not None else None) + dct["type"].append("weakness") + + if related_weaknesses: + dct["child_of"].append( + { + "CWE-" + x.attrib["CWE_ID"] + for x in related_weaknesses + if x.tag == "{http://cwe.mitre.org/cwe-6}Related_Weakness" and x.attrib["Nature"] == "ChildOf" + } + ) + else: + dct["child_of"].append(np.nan) + + assert categories + for category in categories: + assert category + summary = category.find("{http://cwe.mitre.org/cwe-6}Summary") + + dct["cwe_id"].append("CWE-" + category.attrib["ID"]) + dct["cwe_name"].append(category.attrib["Name"]) + dct["cwe_description"].append(summary.text if summary is not None else None) + dct["type"].append("category") + dct["child_of"].append(np.nan) + + cwe_df = pd.DataFrame(dct).set_index("cwe_id") + cwe_df["url"] = cwe_df.index.map(lambda x: "https://cwe.mitre.org/data/definitions/" + x.split("-")[1] + ".html") + cwe_df = cwe_df.replace(r"\n", " ", regex=True) + + if fine_grained: + return df_cwe_relevant, cwe_df + else: + return get_coarse_grained_cwes(df_cwe_relevant, cwe_df), cwe_df + + +def get_coarse_grained_cwes(fine_grained_df: pd.DataFrame, cwe_df: pd.DataFrame) -> pd.DataFrame: + """ + Oddly enough, NVD contains CWEs at different levels of abstraction, which makes it difficult to compare between them. + Among others, some three different CWEs appear in the CVEDataset: CWE-20, CWE-119, CWE-787. Problem is that CWE-787 + is child of CWE-119, which in turn is child of CWE-20. It makes no sense to compute stats of most prevalent CWEs + unless categories are aligned to the top-most level. + + This function aligns the categories to the top-most level. It works in loop. When an iteration is performed without + replacing any CWEs with their parents, the algorithm terminates. + The algorithm inspects every CWE and replaces it with all its parents on condition that they appear in the CVE Dataset. + + :param pd.DataFrame fine_grained_df: First element of the output of `prepare_cwe_df` function + :param pd.DataFrame cwe_df: Second element of the output of `prepare_cwe_df` function + :return pd.DataFrame: DF obtained from CC Dataset, fully exploded to coarse-grained CWEs + """ + all_cwes_in_original_df = set(fine_grained_df.cwe_id.unique()) + parent_dict = cwe_df.child_of.to_dict() + new_set = set(fine_grained_df.cwe_id.unique()) + mapping = {x: {x} for x in new_set} + + while True: + old_set = copy.deepcopy(new_set) + for cwe in old_set: + parents = parent_dict[cwe] + if parents and parents is not np.nan and any(x in all_cwes_in_original_df for x in parents): + new_set.remove(cwe) + new_set.update({x for x in parents if x in all_cwes_in_original_df}) + for val in mapping.values(): + if cwe in val: + val.remove(cwe) + val.update({x for x in parents if x in all_cwes_in_original_df}) + if new_set == old_set: + break + + # Now we should have complete mapping of fine_grained -> coarse_grained CWEs + new_df = fine_grained_df.copy() + new_df.cwe_id = new_df.cwe_id.map(mapping) + + return new_df.explode(column="cwe_id") + + +def get_top_n_cwes( + df: pd.DataFrame, cwe_df: pd.DataFrame, category: str | None = None, eal: str | None = None, n_cwes: int = 10 +) -> pd.DataFrame: + """Fetches top-n CWEs, overall, per category, or per EAL""" + top_n = df.copy() + + if category: + top_n = top_n.loc[top_n.category == category].copy() + if eal: + top_n = top_n.loc[top_n.eal == eal].copy() + + top_n = ( + top_n.cwe_id.value_counts() + .head(n_cwes) + .to_frame() + .rename(columns={"cwe_id": "frequency"}) + .rename_axis("cwe_id") + ) + top_n["cwe_name"] = top_n.index.map(lambda x: cwe_df.loc[x].cwe_name) + top_n["cwe_description"] = top_n.index.map(lambda x: cwe_df.loc[x].cwe_description) + top_n["url"] = top_n.index.map(lambda x: cwe_df.loc[x].url) + top_n["type"] = top_n.index.map(lambda x: cwe_df.loc[x].type) + + return top_n + + +def compute_maintenances_that_come_after_vulns(df: pd.DataFrame) -> pd.DataFrame: + """ + Given pre-processed CCDataset DataFrame (expanded with MU & CVE cols), computes time to fix CVE and earliest CVE after some vuln. + """ + df_fixed = df.loc[(df.n_cves > 0) & (df.n_maintenances > 0)].copy() + df_fixed.maintenance_dates = df_fixed.maintenance_dates.map(lambda x: [y.date() for y in x]) + df_fixed.loc[:, "earliest_maintenance_after_vuln"] = df_fixed.apply(find_earliest_maintenance_after_cve, axis=1) + df_fixed.index.name = "dgst" + return df_fixed + + +def move_fixing_mu_to_directory( + df_fixed: pd.DataFrame, main_df: pd.DataFrame, outdir: str | Path, inpath: str | Path +) -> pd.DataFrame: + """ + Localizes reports of maintenance updates that should fix some vulnerability and copies them into a directory. + df_fixed should be the output of compute_maintenances_that_come_after_vulns method. + """ + fixed_df_index = ( + df_fixed.loc[~df_fixed.earliest_maintenance_after_vuln.isnull()] + .reset_index() + .set_index(["dgst", "earliest_maintenance_after_vuln"]) + .index.to_flat_index() + ) + main_df.maintenance_date = main_df.maintenance_date.map(lambda x: x.date()) + main_prefiltered = main_df.reset_index().set_index(["related_cert_digest", "maintenance_date"]) + mu_filenames = main_prefiltered.loc[main_prefiltered.index.isin(fixed_df_index), "dgst"] + mu_filenames = mu_filenames.map(lambda x: x + ".pdf") + + inpath = Path(inpath) + if not inpath.exists(): + inpath.mkdir() + + for i in mu_filenames: + copyfile(inpath / i, Path(outdir) / i) + + return mu_filenames + + +def plot_dataframe_graph( + data: dict, + label: str, + file_name: str, + density: bool = False, + cumulative: bool = False, + bins: int = 50, + log: bool = True, + show: bool = True, +) -> None: + pd_data = pd.Series(data) + pd_data.hist(bins=bins, label=label, density=density, cumulative=cumulative) + plt.savefig(file_name) + if show: + plt.show() + + if log: + sorted_data = pd_data.value_counts(ascending=True) + + logger.info(sorted_data.where(sorted_data > 1).dropna()) diff --git a/src/sec_certs/utils/parallel_processing.py b/src/sec_certs/utils/parallel_processing.py new file mode 100644 index 00000000..50806a67 --- /dev/null +++ b/src/sec_certs/utils/parallel_processing.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import time +from multiprocessing.pool import ThreadPool +from typing import Any, Callable, Iterable + +from billiard.pool import Pool + +from sec_certs.utils.tqdm import tqdm + + +def process_parallel( + func: Callable, + items: Iterable, + max_workers: int, + callback: Callable | None = None, + use_threading: bool = True, + progress_bar: bool = True, + unpack: bool = False, + progress_bar_desc: str | None = None, +) -> list[Any]: + + pool: Pool | ThreadPool = ThreadPool(max_workers) if use_threading else Pool(max_workers) + results = ( + [pool.apply_async(func, (*i,), callback=callback) for i in items] + if unpack + else [pool.apply_async(func, (i,), callback=callback) for i in items] + ) + + if progress_bar is True and items: + bar = tqdm(total=len(results), desc=progress_bar_desc) + while not all(all_done := [x.ready() for x in results]): + done_count = len(list(filter(lambda x: x, all_done))) + bar.update(done_count - bar.n) + time.sleep(1) + bar.update(len(results) - bar.n) + bar.close() + + pool.close() + pool.join() + pool.terminate() + + return [r.get() for r in results] diff --git a/src/sec_certs/utils/pdf.py b/src/sec_certs/utils/pdf.py new file mode 100644 index 00000000..1d04a697 --- /dev/null +++ b/src/sec_certs/utils/pdf.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import glob +import logging +import subprocess +from datetime import datetime, timedelta, timezone +from functools import reduce +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +import pdftotext +import pikepdf +from PyPDF2 import PdfFileReader +from PyPDF2.generic import BooleanObject, ByteStringObject, FloatObject, IndirectObject, NumberObject, TextStringObject + +from sec_certs import constants as constants +from sec_certs.constants import ( + GARBAGE_ALPHA_CHARS_THRESHOLD, + GARBAGE_AVG_LLEN_THRESHOLD, + GARBAGE_EVERY_SECOND_CHAR_THRESHOLD, + GARBAGE_LINES_THRESHOLD, + GARBAGE_SIZE_THRESHOLD, +) + +logger = logging.getLogger(__name__) + + +def repair_pdf(file: Path) -> None: + """ + Some pdfs can't be opened by PyPDF2 - opening them with pikepdf and then saving them fixes this issue. + By opening this file in a pdf reader, we can already extract number of pages. + + :param file: file name + :return: number of pages in pdf file + """ + pdf = pikepdf.Pdf.open(file, allow_overwriting_input=True) + pdf.save(file) + + +def ocr_pdf_file(pdf_path: Path) -> str: + """ + OCR a PDF file and return its text contents, uses `pdftoppm` and `tesseract`. + + :param pdf_path: The PDF file to OCR. + :return: The text contents. + """ + with TemporaryDirectory() as tmpdir: + tmppath = Path(tmpdir) + ppm = subprocess.run( + ["pdftoppm", pdf_path, tmppath / "image"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + if ppm.returncode != 0: + raise ValueError(f"pdftoppm failed: {ppm.returncode}") + for ppm_path in map(Path, glob.glob(str(tmppath / "image*.ppm"))): + base = ppm_path.with_suffix("") + tes = subprocess.run( + ["tesseract", "-l", "eng+deu+fra", ppm_path, base], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + if tes.returncode != 0: + raise ValueError(f"tesseract failed: {tes.returncode}") + contents = "" + txt_paths = list(glob.glob(str(tmppath / "image*.txt"))) + for txt_path in map(Path, sorted(txt_paths, key=lambda fname: int(fname[6:-4]))): + with txt_path.open("r", encoding="utf-8") as f: + contents += f.read() + return contents + + +def convert_pdf_file(pdf_path: Path, txt_path: Path) -> tuple[bool, bool]: + """ + Convert a PDF tile to text and save it on the `txt_path`. + + :param pdf_path: Path to the to-be-converted PDF file. + :param txt_path: Path to the resulting text file. + :return: A tuple of two results, whether OCR was done and what the complete result + was (OK/NOK). + """ + txt = None + ok = False + ocr = False + try: + with pdf_path.open("rb") as pdf_handle: + pdf = pdftotext.PDF(pdf_handle, "", True) # No password, Raw=True + txt = "".join(pdf) + except Exception as e: + logger.error(f"Error when converting pdf->txt: {e}") + + if txt is None or text_is_garbage(txt): + logger.warning(f"Detected garbage during conversion of {pdf_path}") + ocr = True + try: + txt = ocr_pdf_file(pdf_path) + logger.info(f"OCR OK for {pdf_path}") + except Exception as e: + logger.error(f"Error during OCR of {pdf_path}, using garbage: {e}") + + if txt is not None: + ok = True + with txt_path.open("w", encoding="utf-8") as txt_handle: + txt_handle.write(txt) + + return ocr, ok + + +def parse_pdf_date(dateval: bytes | None) -> datetime | None: + """ + Parse PDF metadata date format: + + ``` + parse_pdf_date(b"D:20110617082321-04'00'") + ``` + into + ``` + datetime.datetime(2011, 6, 17, 8, 23, 21, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=72000))) + ``` + + :param dateval: The date as in the PDF metadata. + :return: The parsed datetime, if successful, else `None`. + """ + if dateval is None: + return None + clean = dateval.decode("utf-8").replace("D:", "") + tz = None + tzoff = None + if "+" in clean: + clean, tz = clean.split("+") + tzoff = 1 + if "-" in clean: + clean, tz = clean.split("-") + tzoff = -1 + elif "Z" in clean: + clean, tz = clean.split("Z") + tzoff = 1 + try: + res_datetime = datetime.strptime(clean, "%Y%m%d%H%M%S") + if tz and tzoff: + tz_datetime = datetime.strptime(tz, "%H'%M'") + delta = tzoff * timedelta(hours=tz_datetime.hour, minutes=tz_datetime.minute) + res_tz = timezone(delta) + res_datetime = res_datetime.replace(tzinfo=res_tz) + return res_datetime + except ValueError: + return None + + +def extract_pdf_metadata(filepath: Path) -> tuple[str, dict[str, Any] | None]: # noqa: C901 + """ + Extract PDF metadata, such as the number of pages, author, title, etc. + + :param filepath: THe path to the PDF. + :return: A tuple of the result code (see constants) and the metadata dictionary. + """ + + def map_metadata_value(val, nope_out=False): + if isinstance(val, BooleanObject): + val = val.value + elif isinstance(val, FloatObject): + val = float(val) + elif isinstance(val, NumberObject): + val = int(val) + elif isinstance(val, IndirectObject) and not nope_out: + # Let's make sure to nope out in case of cycles + val = map_metadata_value(val.getObject(), nope_out=True) + elif isinstance(val, TextStringObject): + val = str(val) + elif isinstance(val, ByteStringObject): + try: + val = val.decode("utf-8") + except UnicodeDecodeError: + val = str(val) + else: + val = str(val) + return val + + def resolve_indirect(val, bound=10): + if isinstance(val, list) and bound: + return [resolve_indirect(v, bound - 1) for v in val] + elif isinstance(val, IndirectObject) and bound: + return resolve_indirect(val.getObject(), bound - 1) + else: + return val + + metadata: dict[str, Any] = dict() + + try: + metadata["pdf_file_size_bytes"] = filepath.stat().st_size + with filepath.open("rb") as handle: + pdf = PdfFileReader(handle, strict=False) + metadata["pdf_is_encrypted"] = pdf.getIsEncrypted() + + # see https://stackoverflow.com/questions/26242952/pypdf-2-decrypt-not-working + if metadata["pdf_is_encrypted"]: + pikepdf.open(filepath, allow_overwriting_input=True).save() + + with filepath.open("rb") as handle: + pdf = PdfFileReader(handle, strict=False) + metadata["pdf_number_of_pages"] = pdf.getNumPages() + pdf_document_info = pdf.getDocumentInfo() + + if pdf_document_info is None: + raise ValueError("PDF metadata unavailable") + + for key, val in pdf_document_info.items(): + metadata[str(key)] = map_metadata_value(val) + + # Get the hyperlinks in the PDF + annots = [page.get("/Annots", []) for page in pdf.pages] + annots = reduce(lambda x, y: x + y, map(resolve_indirect, annots)) + links = set() + for annot in annots: + try: + A = resolve_indirect(annot.get("/A", {})) + link = resolve_indirect(A.get("/URI")) + if link: + links.add(map_metadata_value(link)) + except Exception: + pass + metadata["pdf_hyperlinks"] = links + + except Exception as e: + relative_filepath = "/".join(str(filepath).split("/")[-4:]) + error_msg = f"Failed to read metadata of {relative_filepath}, error: {e}" + logger.error(error_msg) + return error_msg, None + + return constants.RETURNCODE_OK, metadata + + +def text_is_garbage(text: str) -> bool: + """ + Detect whether the given text is "garbage". A series of tests is applied, + using the number of lines, average line length, total size, every second character on a line + and the ratio of alphanumeric characters. + + :param text: The tested text. + :return: Whether the text is a "garbage" result of pdftotext conversion. + """ + size = len(text) + content_len = 0 + lines = 0 + every_second = 0 + alpha_len = len("".join(filter(str.isalpha, text))) + for line in text.splitlines(): + content_len += len(line) + lines += 1 + if len(set(line[1::2])) > 1: + every_second += 1 + + if lines: + avg_line_len = content_len / lines + else: + avg_line_len = 0 + if size: + alpha = alpha_len / size + else: + alpha = 0 + + # If number of lines is small, this is garbage. + if lines < GARBAGE_LINES_THRESHOLD: + return True + # If the file size is small, this is garbage. + if size < GARBAGE_SIZE_THRESHOLD: + return True + # If the average length of a line is small, this is garbage. + if avg_line_len < GARBAGE_AVG_LLEN_THRESHOLD: + return True + # If there a small amount of lines that have more than one character at every second character, this is garbage. + # This detects the ANSSI spacing issues. + if every_second < GARBAGE_EVERY_SECOND_CHAR_THRESHOLD: + return True + # If there is a small ratio of alphanumeric chars to all chars, this is garbage. + if alpha < GARBAGE_ALPHA_CHARS_THRESHOLD: + return True + return False diff --git a/src/sec_certs/utils/sanitization.py b/src/sec_certs/utils/sanitization.py new file mode 100644 index 00000000..2f9cd046 --- /dev/null +++ b/src/sec_certs/utils/sanitization.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import html +import logging +from datetime import date + +import numpy as np +import pandas as pd +from bs4 import NavigableString + +logger = logging.getLogger(__name__) + + +def sanitize_navigable_string(string: NavigableString | str | None) -> str | None: + if not string: + return None + return str(string).strip().replace("\xad", "").replace("\xa0", "") + + +def sanitize_link(record: str | None) -> str | None: + if not record: + return None + return record.replace(":443", "").replace(" ", "%20").replace("http://", "https://") + + +def sanitize_date(record: pd.Timestamp | date | np.datetime64) -> date | None: + if pd.isnull(record): + return None + elif isinstance(record, pd.Timestamp): + return record.date() + elif isinstance(record, (date, type(None))): + return record + raise ValueError("Unsupported type given as input") + + +def sanitize_string(record: str) -> str: + # There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I &#38; J' that has to be unescaped twice + string = html.unescape(html.unescape(record)).replace("\n", "") + return " ".join(string.split()) + + +def sanitize_security_levels(record: str | set[str]) -> set[str]: + if isinstance(record, str): + record = set(record.split(",")) + return record - {"Basic", "ND-PP", "PP\xa0Compliant", "None", "Medium"} + + +def sanitize_protection_profiles(record: str) -> list: + if not record: + return [] + return record.split(",") diff --git a/src/sec_certs/utils/tables.py b/src/sec_certs/utils/tables.py new file mode 100644 index 00000000..29def971 --- /dev/null +++ b/src/sec_certs/utils/tables.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import logging +import re +from pathlib import Path + +from sec_certs.cert_rules import FIPS_LIST_OF_TABLES + +logger = logging.getLogger(__name__) + + +def parse_list_of_tables(txt: str) -> set[int]: + """ + Parses list of tables in policy txt, returns page numbers of tables that mention algorithms + """ + rr = re.compile(r"^.+?(?:[Ff]unction|[Aa]lgorithm|[Ss]ecurity [Ff]unctions?).+?(?P\d+)$", re.MULTILINE) + return {int(m.group("page_num")) for m in rr.finditer(txt)} + + +def get_table_rich_page_numbers_from_footer(file_text: str) -> set[int]: + """ + Parses page numbers of policy txt pages that may contain tables with algorithm data + """ + current_page = 1 + pages = set() + + for line in file_text.split("\n"): + if "\f" in line: + current_page += 1 + if line.startswith("Table ") or line.startswith("Exhibit"): + pages.add(current_page) + pages.add(current_page + 1) + if current_page > 2: + pages.add(current_page - 1) + + for page in pages: + if page > current_page - 1: + return pages - {page} + + return pages + + +def find_pages_with_tables(txt_filepath: Path) -> set[int]: + """ + Identifies pages in txt file that may contain tables. Return their page numbers. + """ + with txt_filepath.open("r", encoding="utf-8") as handle: + txt = handle.read() + + # Parse page numbers from list of tables if available + # Else look for "Table" in text and \f representing footer, then extract page number from footer + if list_of_tables := FIPS_LIST_OF_TABLES.search(txt): + result = parse_list_of_tables(list_of_tables.group()) + else: + result = get_table_rich_page_numbers_from_footer(txt) + + return result if result else set() + + +def get_algs_from_table(dataframe_text: str) -> set[str]: + reg = r"(?:#?\s?|(?:Cert)\.?[^. ]*?\s?)(?:[CcAa]\s)?(?P[CcAa]? ?\d+)" + return {m.group() for m in re.finditer(reg, dataframe_text)} diff --git a/src/sec_certs/utils/tqdm.py b/src/sec_certs/utils/tqdm.py new file mode 100644 index 00000000..77eeae94 --- /dev/null +++ b/src/sec_certs/utils/tqdm.py @@ -0,0 +1,9 @@ +from tqdm import tqdm as tqdm_original + +from sec_certs.config.configuration import config + + +def tqdm(*args, **kwargs): + if "disable" in kwargs: + return tqdm_original(*args, **kwargs) + return tqdm_original(*args, **kwargs, disable=not config.enable_progress_bars) -- cgit v1.3.1