From 10b3b57c8df3e467df2f8fbc617c25a6c0d578d5 Mon Sep 17 00:00:00 2001
From: Adam Janovsky
Date: Fri, 20 May 2022 10:37:09 +0200
Subject: simplify FIPS,CC examples
---
examples/cc_oop_demo.py | 37 +++++++-----------------
examples/fips_oop_demo.py | 73 +++++++++++++----------------------------------
2 files changed, 31 insertions(+), 79 deletions(-)
(limited to 'examples')
diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py
index 46f4d882..bf4ed31c 100644
--- a/examples/cc_oop_demo.py
+++ b/examples/cc_oop_demo.py
@@ -1,6 +1,5 @@
import logging
from datetime import datetime
-from pathlib import Path
from sec_certs.config.configuration import config
from sec_certs.dataset.common_criteria import CCDataset
@@ -17,40 +16,26 @@ def main():
logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler])
start = datetime.now()
- # Create empty dataset
- dset = CCDataset({}, Path("./debug_dataset"), "cc_full_dataset", "sample dataset description")
+ # Full processing pipeline of Common Criteria dataset
+ dset = CCDataset()
+ dset.get_certs_from_web()
+ dset.process_protection_profiles()
+ dset.download_all_pdfs()
+ dset.convert_all_pdfs()
+ dset.analyze_certificates() # calls dset._extract_data() and dset._compute_heuristics()
- # Load metadata for certificates from CSV and HTML sources
- dset.get_certs_from_web(to_download=True)
+ # Other useful API below
# explicitly dump to json
- dset.to_json(dset.json_path)
-
- # Retrieve protection profile IDs
- dset.process_protection_profiles()
+ # dset.to_json(dset.json_path)
# Load dataset from JSON
- new_dset = CCDataset.from_json("./debug_dataset/cc_full_dataset.json")
- assert dset == new_dset
-
- # Download pdfs and update json
- dset.download_all_pdfs()
-
- # Convert pdfs to text and update json
- dset.convert_all_pdfs()
-
- # Extract data from txt files and update json
- dset._extract_data()
+ # new_dset = CCDataset.from_json("./debug_dataset/cc_full_dataset.json")
+ # assert dset == new_dset
# transform to pandas DataFrame
# df = dset.to_pandas()
- # Compute heuristics on the dataset
- dset._compute_heuristics()
-
- # Compute related cves
- # dset.compute_related_cves()
-
end = datetime.now()
logger.info(f"The computation took {(end-start)} seconds.")
diff --git a/examples/fips_oop_demo.py b/examples/fips_oop_demo.py
index a84b1ac9..d31db4b2 100644
--- a/examples/fips_oop_demo.py
+++ b/examples/fips_oop_demo.py
@@ -1,71 +1,38 @@
import logging
from datetime import datetime
-from pathlib import Path
-
-import click
from sec_certs.config.configuration import config
from sec_certs.dataset.fips import FIPSDataset
-from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset
logger = logging.getLogger(__name__)
-@click.command()
-@click.option("--config-file", help="Path to config file")
-@click.option("--json-file", help="Path to dataset json file")
-@click.option("--no-download-algs", help="Redo scan of html files", is_flag=True)
-@click.option("--redo-web-scan", help="Redo scan of PDF files", is_flag=True)
-@click.option("--redo-keyword-scan", help="Don't download algs", is_flag=True)
-@click.option(
- "--higher-precision-results",
- help="Redo table search for certificates with high error rate. Behaviour undefined if used on a newly instantiated dataset.",
- is_flag=True,
-)
-def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_scan, higher_precision_results):
- logging.basicConfig(level=logging.INFO)
+def main():
+ file_handler = logging.FileHandler(config.log_filepath)
+ stream_handler = logging.StreamHandler()
+ formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+ file_handler.setFormatter(formatter)
+ stream_handler.setFormatter(formatter)
+ logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler])
start = datetime.now()
- # Load config
- config.load(config_file if config_file else "../sec_certs/settings.yaml")
-
- # Create empty dataset
- dset = FIPSDataset({}, Path("../fips_dataset"), "sample_dataset", "sample dataset description")
-
- # this is for creating test dataset, usually with small number of pdfs
- # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing')
-
- # Load metadata for certificates from CSV and HTML sources
- dset.get_certs_from_web(redo=redo_web_scan)
-
- logger.info(f"Finished parsing. Have dataset with {len(dset)} certificates.")
- logger.info(f"Dataset saved to {dset.root_dir}/fips_full_dataset.json")
-
- logger.info("Converting pdfs")
+ # Full processing of FIPS dataset, fresh run
+ dset = FIPSDataset()
+ dset.get_certs_from_web()
dset.convert_all_pdfs()
+ dset.pdf_scan()
+ dset.extract_certs_from_tables()
+ dset.finalize_results()
- logger.info("Extracting keywords now.")
- dset.pdf_scan(redo=redo_keyword_scan)
-
- logger.info(f"Finished extracting certificates for {len(dset.certs)} items.")
-
- logger.info("Searching for tables in pdfs")
-
- not_decoded_files = dset.extract_certs_from_tables(higher_precision_results)
-
- logger.info(f"Done. Files not decoded: {not_decoded_files}")
- logger.info("Parsing algorithms")
- if not no_download_algs:
- aset = FIPSAlgorithmDataset({}, Path(dset.root_dir / "web/algorithms"), "algorithms", "sample algs")
- aset.get_certs_from_web()
- logger.info(f"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.")
-
- dset.algorithms = aset
+ # TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/210 and refactor this
+ # if not no_download_algs:
+ # aset.get_certs_from_web()
+ # logger.info(f"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.")
+ # dset.algorithms = aset
- logger.info("finalizing results.")
- dset.finalize_results()
+ # TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/211 and uncomment
+ # dset.plot_graphs(show=False)
- dset.plot_graphs(show=False)
end = datetime.now()
logger.info(f"The computation took {(end - start)} seconds.")
--
cgit v1.3.1
From 9db545dbb2b644a01b8e49b4b2506ea03ec36b00 Mon Sep 17 00:00:00 2001
From: Adam Janovsky
Date: Fri, 20 May 2022 17:20:48 +0200
Subject: Docs: add some examples
---
docs/api/dataset.md | 18 +++
docs/api/sample.md | 18 +++
docs/index.md | 19 ++-
docs/quickstart.md | 10 +-
examples/cc_oop_demo.py | 44 -------
notebooks/examples/common_criteria.ipynb | 198 ++++++++++++++++++++++++++++---
notebooks/examples/fips.ipynb | 139 ++++++++++++++++++++--
notebooks/examples/model.ipynb | 76 ++++++++++++
sec_certs/dataset/fips.py | 5 +-
9 files changed, 450 insertions(+), 77 deletions(-)
create mode 100644 docs/api/dataset.md
create mode 100644 docs/api/sample.md
delete mode 100644 examples/cc_oop_demo.py
create mode 100644 notebooks/examples/model.ipynb
(limited to 'examples')
diff --git a/docs/api/dataset.md b/docs/api/dataset.md
new file mode 100644
index 00000000..ed92b31e
--- /dev/null
+++ b/docs/api/dataset.md
@@ -0,0 +1,18 @@
+# Dataset package
+
+```{eval-rst}
+.. automodule:: sec_certs.dataset
+ :no-members:
+```
+
+```{tip}
+The examples related to this package can be found at [common criteria notebook](./../notebooks/examples/common_criteria.ipynb) and [fips notebook](./../notebooks/examples/fips.ipynb).
+```
+
+## CCDataset
+
+```{eval-rst}
+.. currentmodule:: sec_certs.dataset
+.. autoclass:: CCDataset
+ :members:
+```
\ No newline at end of file
diff --git a/docs/api/sample.md b/docs/api/sample.md
new file mode 100644
index 00000000..29266f90
--- /dev/null
+++ b/docs/api/sample.md
@@ -0,0 +1,18 @@
+# Sample package
+
+```{eval-rst}
+.. automodule:: sec_certs.sample
+ :no-members:
+```
+
+```{tip}
+The examples related to this package can be found at [common criteria notebook](./../notebooks/examples/common_criteria.ipynb) and [fips notebook](./../notebooks/examples/fips.ipynb).
+```
+
+## CommonCriteriaCert
+
+```{eval-rst}
+.. currentmodule:: sec_certs.sample
+.. autoclass:: CommonCriteriaCert
+ :members:
+```
\ No newline at end of file
diff --git a/docs/index.md b/docs/index.md
index 3f44633b..8949debe 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -4,6 +4,13 @@ Welcome to the technical documentation of *sec-certs* tool for the data analysis
There are three main parts of this documentation. *User's guide* describes high-level use of our tool. Driven by this knowledge, you can progress to *Notebook examples* that showcase most of the API that we use in the form of Jupyter notebooks. The documentation currently does not have all modules documented with `autodoc`, so for the API reference, you must directly inspect the [sec_certs](https://github.com/crocs-muni/sec-certs/tree/main/sec_certs) module. If you want, you can run the notebooks as they are stored in the [project repository](https://github.com/crocs-muni/sec-certs/tree/main/notebooks). If you are interested in contributing to our project or in other aspects of our development, you can consult the relevant *GitHub artifacts*
+```{button-ref} quickstart
+:align: center
+:color: primary
+:ref-type: myst
+Show me Quickstart!
+```
+
```{admonition} Launch notebooks in MyBinder
Each of the notebooks can be launched interactively in MyBinder by clicking on 🚀 icon (top-right corner).
```
@@ -11,6 +18,7 @@ Each of the notebooks can be launched interactively in MyBinder by clicking on
```{toctree}
:hidden:
:caption: Navigation
+:maxdepth: 1
Seccerts homepage
Seccerts docs
GitHub repo
@@ -18,7 +26,9 @@ Seccerts PyPi
```
```{toctree}
+:hidden: True
:caption: User's guide
+:maxdepth: 1
installation.md
quickstart.md
tutorial.md
@@ -26,6 +36,8 @@ tutorial.md
```{toctree}
:caption: Notebook examples
+:hidden: True
+:maxdepth: 1
notebooks/examples/common_criteria.ipynb
notebooks/examples/fips.ipynb
notebooks/examples/model.ipynb
@@ -33,11 +45,16 @@ notebooks/examples/model.ipynb
```{toctree}
:caption: API
+:hidden: True
+:maxdepth: 1
+api/sample.md
+api/dataset.md
api/model.md
```
```{toctree}
-:maxdepth: 2
+:maxdepth: 1
+:hidden: True
:caption: GitHub artifacts
readme.md
contributing.md
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 14121212..721e1341 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -6,6 +6,8 @@
1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)).
2. Use
```python
+from sec-certs.dataset import CCDataset
+
dset = CCDataset.from_web_latest()
```
to obtain to obtain freshly processed dataset from [seccerts.org](https://seccerts.org).
@@ -17,6 +19,8 @@ to obtain to obtain freshly processed dataset from [seccerts.org](https://seccer
1. Install the latest version with `pip install -U sec-certs` (see [installation](installation.md)).
2. Use
```python
+from sec-certs.dataset import FIPSDataset
+
dset = FIPSDataset.from_web_latest()
```
to obtain to obtain freshly processed dataset from [seccerts.org](https://seccerts.org).
@@ -45,4 +49,8 @@ $ fips-certs new-run
:::
::::
-This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`.
\ No newline at end of file
+This script takes a long time to run (few hours) and will create `./cc_dset` or `./fips_dset` directory. To see all options, call the entrypoint with `--help`.
+
+:::{hint}
+If you installed the docker image, use `docker run -it sec-certs bash` to run the container interactively.
+:::
diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py
deleted file mode 100644
index 7876c71f..00000000
--- a/examples/cc_oop_demo.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import logging
-from datetime import datetime
-
-from sec_certs.config.configuration import config
-from sec_certs.dataset import CCDataset
-
-logger = logging.getLogger(__name__)
-
-
-def main():
- file_handler = logging.FileHandler(config.log_filepath)
- stream_handler = logging.StreamHandler()
- formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
- file_handler.setFormatter(formatter)
- stream_handler.setFormatter(formatter)
- logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler])
- start = datetime.now()
-
- # Full processing pipeline of Common Criteria dataset
- dset = CCDataset()
- dset.get_certs_from_web()
- dset.process_protection_profiles()
- dset.download_all_pdfs()
- dset.convert_all_pdfs()
- dset.analyze_certificates() # calls dset._extract_data() and dset._compute_heuristics()
-
- # Other useful API below
-
- # explicitly dump to json
- # dset.to_json(dset.json_path)
-
- # Load dataset from JSON
- # new_dset = CCDataset.from_json("./debug_dataset/cc_full_dataset.json")
- # assert dset == new_dset
-
- # transform to pandas DataFrame
- # df = dset.to_pandas()
-
- end = datetime.now()
- logger.info(f"The computation took {(end-start)} seconds.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/notebooks/examples/common_criteria.ipynb b/notebooks/examples/common_criteria.ipynb
index 4aab6c61..83a56379 100644
--- a/notebooks/examples/common_criteria.ipynb
+++ b/notebooks/examples/common_criteria.ipynb
@@ -4,36 +4,202 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Common Criteria dataset class\n",
+ "# Common Criteria example\n",
"\n",
- "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset"
+ "This notebook illustrates basic functionality with the `CCDataset` class that holds Common Criteria dataset and of its sample `CommonCriteriaCert`"
]
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
- "from sec_certs.dataset.common_criteria import CCDataset"
+ "from sec_certs.dataset.common_criteria import CCDataset\n",
+ "from sec_certs.dataset.common_criteria import CommonCriteriaCert\n",
+ "import pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Get fresh dataset snapshot from mirror"
]
},
{
"cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "4985\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"dset = CCDataset.from_web_latest()\n",
- "print(len(dset))"
+ "print(len(dset)) # Print number of certificates in the dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Do some basic dataset serialization"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Dump dataset into json and load it back\n",
+ "dset.to_json(\"./cc_dset.json\")\n",
+ "new_dset: CCDataset = CCDataset.from_json(\"./cc_dset.json\")\n",
+ "assert dset == new_dset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Turn dataset into Pandas DataFrame\n",
+ "df = dset.to_pandas()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Simple dataset manipulation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Iterate over certificates in dataset\n",
+ "for cert in dset:\n",
+ " pass\n",
+ "\n",
+ "# Get certificates produced by Infineon manufacturer\n",
+ "infineon_certs = [x for x in dset if \"Infineon\" in x.manufacturer]\n",
+ "df_infineon = df.loc[df.manufacturer.str.contains(\"Infineon\", case=False)]\n",
+ "\n",
+ "# Get certificates with some CVE\n",
+ "vulnerable_certs = [x for x in dset if x.heuristics.related_cves]\n",
+ "df_vulnerable = df.loc[~df.related_cves.isna()]\n",
+ "\n",
+ "# Show CVE ids of some vulnerable certificate\n",
+ "print(f\"{vulnerable_certs[0].heuristics.related_cves=}\")\n",
+ "\n",
+ "# Get certificates from 2015 and newer\n",
+ "df_2015_and_newer = df.loc[df.year_from > 2014]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Plot distribution of years of certification\n",
+ "df.year_from.value_counts().sort_index().plot.line()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Dissect single certificate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Select a certificate and print some attributes\n",
+ "cert: CommonCriteriaCert = dset[\"bad93fb821395db2\"]\n",
+ "print(f\"{cert.name=}\")\n",
+ "print(f\"{cert.heuristics.cpe_matches=}\")\n",
+ "print(f\"{cert.heuristics.report_references.directly_referencing=}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vulnerable_certs = [x for x in dset if x.heuristics.related_cves]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Serialize single certificate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cert.to_json(\"./cert.json\")\n",
+ "new_cert = cert.from_json(\"./cert.json\")\n",
+ "assert cert == new_cert\n",
+ "\n",
+ "# Serialize as Pandas series\n",
+ "ser = pd.Series(cert.pandas_tuple, index=cert.pandas_columns)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Assign dataset with CPE records and compute vulnerabilities\n",
+ "\n",
+ "*Note*: The data is already computed on dataset obtained with `from_web_latest()`, this is just for illustration. \n",
+ "*Note*: This may likely not run in Binder, as the corresponding `CVEDataset` and `CPEDataset` instances take a lot of memory."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Automatically match CPEs and CVEs\n",
+ "_, cpe_dset, _ = dset.compute_cpe_heuristics()\n",
+ "dset.compute_related_cves()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Create new dataset and fully process it\n",
+ "\n",
+ "*Warning*: It's not good idea to run this from notebook. It may take several hours to finnish. We recommend using `from_web_latest()` or turning this into a Python script."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dset = CCDataset()\n",
+ "dset.get_certs_from_web()\n",
+ "dset.process_protection_profiles()\n",
+ "dset.download_all_pdfs()\n",
+ "dset.convert_all_pdfs()\n",
+ "dset.analyze_certificates()"
]
}
],
diff --git a/notebooks/examples/fips.ipynb b/notebooks/examples/fips.ipynb
index ab20ee93..28eb2b1a 100644
--- a/notebooks/examples/fips.ipynb
+++ b/notebooks/examples/fips.ipynb
@@ -11,30 +11,143 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
- "from sec_certs.dataset.fips import FIPSDataset"
+ "from sec_certs.dataset.fips import FIPSDataset\n",
+ "from sec_certs.sample import FIPSCertificate"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Get fresh dataset snapshot from mirror"
]
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "4985\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "dset = FIPSDataset.from_web_latest()\n",
+ "dset: FIPSDataset = FIPSDataset.from_web_latest()\n",
"print(len(dset))"
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Do some basic dataset serialization"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Dump dataset into json and load it back\n",
+ "dset.to_json(\"./fips_dataset.json\")\n",
+ "new_dset = FIPSDataset.from_json(\"./fips_dataset.json\")\n",
+ "assert dset == new_dset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Simple dataset manipulation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get certificates from a single manufacturer\n",
+ "cisco_certs = [cert for cert in dset if \"Cisco\" in cert.web_scan.vendor]\n",
+ "\n",
+ "# Get certificates with some CVE\n",
+ "vulnerable_certs = [cert for cert in dset if cert.heuristics.related_cves]\n",
+ "\n",
+ "# Show CVE ids of some vulnerable certificate\n",
+ "print(f\"{vulnerable_certs[0].heuristics.related_cves=}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Dissect single certificate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Select a certificate and print some attributes\n",
+ "cert: FIPSCertificate = dset[\"542cacae1d41132a\"]\n",
+ "\n",
+ "print(f\"{cert.web_scan.module_name=}\")\n",
+ "print(f\"{cert.heuristics.cpe_matches=}\")\n",
+ "print(f\"{cert.web_scan.level=}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Serialize single certificate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cert.to_json(\"./cert.json\")\n",
+ "new_cert: FIPSCertificate = FIPSCertificate.from_json(\"./cert.json\")\n",
+ "assert new_cert == cert"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Create new dataset and fully process it\n",
+ "\n",
+ "*Warning*: It's not good idea to run this from notebook. It may take several hours to finnish. We recommend using `from_web_latest()` or turning this into a Python script."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dset = FIPSDataset()\n",
+ "dset.get_certs_from_web()\n",
+ "dset.convert_all_pdfs()\n",
+ "dset.pdf_scan()\n",
+ "dset.extract_certs_from_tables()\n",
+ "dset.finalize_results()\n",
+ "\n",
+ "# TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/210 and refactor this\n",
+ "# if not no_download_algs:\n",
+ "# aset.get_certs_from_web()\n",
+ "# logger.info(f\"Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.\")\n",
+ "# dset.algorithms = aset\n",
+ "\n",
+ "# TODO: Resolve https://github.com/crocs-muni/sec-certs/issues/211 and uncomment\n",
+ "# dset.plot_graphs(show=False)"
+ ]
}
],
"metadata": {
diff --git a/notebooks/examples/model.ipynb b/notebooks/examples/model.ipynb
new file mode 100644
index 00000000..370b13cd
--- /dev/null
+++ b/notebooks/examples/model.ipynb
@@ -0,0 +1,76 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Model\n",
+ "\n",
+ "This notebook illustrates basic functionality with the `model` package that apply complex transformations on certificates.\n",
+ "\n",
+ "*Note*: You probably don't need to use this. Instead, you should use `CCDataset` or `FIPSDataset` classes to handle the transformations for yourself."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sec_certs.dataset.common_criteria import CCDataset\n",
+ "from sec_certs.model import SARTransformer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dset: CCDataset = CCDataset.from_web_latest()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## SARTransformer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "transformer = SARTransformer().fit(dset.certs.values())\n",
+ "extracted_sars = {x.dgst: transformer.transform_single_cert(x) for x in dset}"
+ ]
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "6386d1612879d92d026c363e7667e428bc38d86c5a080d58c3d70e7cd43df67d"
+ },
+ "kernelspec": {
+ "display_name": "Python 3.8.1 ('certsvenv': venv)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.1"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py
index 2defd9f1..8cef1520 100644
--- a/sec_certs/dataset/fips.py
+++ b/sec_certs/dataset/fips.py
@@ -254,8 +254,9 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType):
len(dset),
len(dset.algorithms) if dset.algorithms is not None else 0,
)
- logger.info("The dataset does not contain the results of the dependency analysis - calculating them now...")
- dset.finalize_results()
+ # TODO: Fixme, this is really costly
+ # logger.info("The dataset does not contain the results of the dependency analysis - calculating them now...")
+ # dset.finalize_results()
return dset
def set_local_paths(self) -> None:
--
cgit v1.3.1