Merge branch 'main' into feat-fips-references

14 files changed, 178 insertions, 144 deletions

diff --git a/README.md b/README.md
index bf6a58ea..a5a18a10 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,9 @@
 
 ![](docs/_static/logo.png)
 
-A tool for data scraping and analysis of security certificates from Common Criteria and FIPS 140-2/3 frameworks. This project is developed by the [Centre for Research On Cryptography and Security](https://crocs.fi.muni.cz) at Masaryk University, Czech Republic.
+A tool for data scraping and analysis of security certificates from Common Criteria and FIPS 140-2/3 frameworks.
+
+<!-- This project is developed by the [Centre for Research On Cryptography and Security](https://crocs.fi.muni.cz) at Masaryk University, Czech Republic. -->
 
 [![Website](https://img.shields.io/website?down_color=red&down_message=offline&style=flat-square&up_color=SpringGreen&up_message=online&url=https%3A%2F%2Fseccerts.org)](https://seccerts.org)
 [![Website](https://img.shields.io/website?label=docs&down_color=red&down_message=offline&style=flat-square&up_color=SpringGreen&up_message=online&url=https%3A%2F%2Fseccerts.org/docs/index.html)](https://seccerts.org/docs/index.html)
@@ -58,8 +60,8 @@ df_2015_and_newer = df.loc[df.year_from > 2014]
 df.year_from.value_counts().sort_index().plot.line()
 ```
 
-## Authors
+<!-- ## Authors
 
 This work is being done at [CRoCS MUNI](https://crocs.fi.muni.cz/) by Adam Janovsky, Jan Jancar, Petr Svenda, Jiri Michalik, Lukasz Chmielewski and other contributors. This work was supported by the Internal grant agency of Masaryk University, CZ.02.2.69/0.0/0.0/19_073/0016943.
 
-![](docs/_static/logolink_OP_VVV_hor_barva_eng.jpg)
\ No newline at end of file
+![](docs/_static/logolink_OP_VVV_hor_barva_eng.jpg) -->
diff --git a/docs/conf.py b/docs/conf.py
index 562491f9..f21ee733 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -18,7 +18,8 @@ from importlib.metadata import version as get_version
 # -- Project information -----------------------------------------------------
 
 project = "sec-certs"
-copyright = "CRoCS MUNI | 2020-2023"
+copyright = "Anonymized | 2020-2023"
+# copyright = "CRoCS MUNI | 2020-2023"
 
 # Note thas this inference won't work from Docker: https://github.com/pypa/setuptools_scm/#usage-from-docker
 release = ".".join(get_version("sec-certs").split(".")[:3])
diff --git a/docs/index.md b/docs/index.md
index 448efbeb..44d64abe 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -21,8 +21,7 @@ Each of the notebooks can be launched interactively in MyBinder by clicking on ð
 :maxdepth: 1
 Seccerts homepage <https://seccerts.org/>
 Seccerts docs <https://seccerts.org/docs>
-GitHub repo <https://github.com/crocs-muni/sec-certs>
-Seccerts PyPi <https://pypi.org/project/sec-certs/>
+GitHub repo <https://anonymous.4open.science/r/sec-certs-7A92>
 ```
 
 ```{toctree}
diff --git a/notebooks/fips/in_process.ipynb b/notebooks/fips/in_process.ipynb
index d127d854..4e0be8cb 100644
--- a/notebooks/fips/in_process.ipynb
+++ b/notebooks/fips/in_process.ipynb
@@ -34,16 +34,20 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "fips = FIPSDataset.from_web_latest()\n"
-   ],
+   "id": "a3faaf17",
    "metadata": {
     "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "fips = FIPSDataset.from_web_latest()\n"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -78,9 +82,7 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "776323d7",
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "iut_melt = iut_global_df.melt(\"timestamp\", var_name=\"type\", value_name=\"count\")\n",
@@ -105,9 +107,7 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "97b726d6",
-   "metadata": {
-    "scrolled": false
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "def iut_key(entry):\n",
@@ -126,7 +126,7 @@
     "\n",
     "#iut_local_df = pd.DataFrame([(entry[0], entry[1], entry[2], iut_first_seen[entry], iut_last_seen[entry], iut_last_seen[entry] == snapshot_date) for entry in iut_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"first_seen\", \"last_seen\", \"present\"))\n",
     "iut_local_df = pd.DataFrame([(entry.module_name, entry.vendor_name, entry.standard, entry.iut_date, iut_first_seen[entry], iut_last_seen[entry], iut_last_seen[entry] == snapshot_date) for entry in iut_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"iut_date\", \"first_seen\", \"last_seen\", \"present\"))\n",
-    "iut_local_df = iut_local_df.astype({\"standard\": \"category\", \"iut_date\": \"datetime64\", \"first_seen\": \"datetime64\", \"last_seen\": \"datetime64\", \"present\": \"bool\"}).fillna(value=np.nan)\n",
+    "iut_local_df = iut_local_df.astype({\"standard\": \"category\", \"iut_date\": \"datetime64[ns]\", \"first_seen\": \"datetime64[ns]\", \"last_seen\": \"datetime64[ns]\", \"present\": \"bool\"}).fillna(value=np.nan)\n",
     "iut_local_df[\"seen_for_iut\"] = (iut_local_df.last_seen - iut_local_df.iut_date).dt.days\n",
     "iut_local_df[\"seen_for\"] = (iut_local_df.last_seen - iut_local_df.first_seen).dt.days\n",
     "\n",
@@ -171,9 +171,7 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "fbfbf88e",
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "hist = sns.histplot(data=iut_local_df[~iut_local_df.present], x=\"seen_for_iut\", hue=\"standard\")"
@@ -201,30 +199,38 @@
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "### IUT - Certificate mapping"
-   ],
+   "id": "e2e5805f",
    "metadata": {
     "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "### IUT - Certificate mapping"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "first_snapshot = iut_dset.snapshots[-1]\n",
-    "matches = FIPSProcessMatcher.match_snapshot(first_snapshot, fips)"
-   ],
+   "id": "48f49012",
    "metadata": {
     "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "first_snapshot = iut_dset.snapshots[-1]\n",
+    "matches = FIPSProcessMatcher.match_snapshot(first_snapshot, fips)"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -303,7 +309,7 @@
     "\n",
     "#mip_local_df = pd.DataFrame([(entry[0], entry[1], entry[2], entry[3], mip_first_seen[entry], mip_last_seen[entry], mip_last_seen[entry] == snapshot_date) for entry in iut_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"status\", first_seen\", \"last_seen\", \"present\"))\n",
     "mip_local_df = pd.DataFrame([(entry.module_name, entry.vendor_name, entry.standard, entry.status, entry.status_since, mip_first_seen[entry], mip_last_seen[entry], mip_last_seen[entry] == snapshot_date) for entry in mip_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"status\", \"status_since\", \"first_seen\", \"last_seen\", \"present\"))\n",
-    "mip_local_df = mip_local_df.astype({\"standard\": \"category\", \"status\": \"category\", \"status_since\": \"datetime64\", \"first_seen\": \"datetime64\", \"last_seen\": \"datetime64\", \"present\": \"bool\"}).fillna(value=np.nan)\n",
+    "mip_local_df = mip_local_df.astype({\"standard\": \"category\", \"status\": \"category\", \"status_since\": \"datetime64[ns]\", \"first_seen\": \"datetime64[ns]\", \"last_seen\": \"datetime64[ns]\", \"present\": \"bool\"}).fillna(value=np.nan)\n",
     "mip_local_df[\"seen_for_status\"] = (mip_local_df.last_seen - mip_local_df.status_since).dt.days\n",
     "mip_local_df[\"seen_for\"] = (mip_local_df.last_seen - mip_local_df.first_seen).dt.days\n",
     "mip_local_df.head()"
@@ -386,35 +392,43 @@
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "### MIP - Certificate matching"
-   ],
+   "id": "b1ce1208",
    "metadata": {
     "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "### MIP - Certificate matching"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "first_snapshot = mip_dset.snapshots[-1]\n",
-    "matches = FIPSProcessMatcher.match_snapshot(first_snapshot, fips)"
-   ],
+   "id": "c692ad3f",
    "metadata": {
     "collapsed": false,
+    "jupyter": {
+     "outputs_hidden": false
+    },
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "first_snapshot = mip_dset.snapshots[-1]\n",
+    "matches = FIPSProcessMatcher.match_snapshot(first_snapshot, fips)"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.8.13 ('venv': venv)",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -428,7 +442,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.13"
+   "version": "3.11.3"
   },
   "vscode": {
    "interpreter": {
@@ -438,4 +452,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
\ No newline at end of file
+}
diff --git a/pyproject.toml b/pyproject.toml
index 7b7715f0..e26334d6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -59,6 +59,7 @@
     "scipy>=1.9.0",
     "networkx",
     "pydantic",
+    "psutil",
   ]
 
   [project.optional-dependencies]
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index f8046840..f4dd8b6d 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -1,9 +1,5 @@
 alabaster==0.7.12
     # via sphinx
-appnope==0.1.3
-    # via
-    #   ipykernel
-    #   ipython
 asttokens==2.2.1
     # via stack-data
 attrs==22.1.0
@@ -77,8 +73,6 @@ docutils==0.17.1
     #   sphinx
 entrypoints==0.4
     # via jupyter-client
-exceptiongroup==1.0.4
-    # via pytest
 executing==1.2.0
     # via stack-data
 fastjsonschema==2.16.2
@@ -103,9 +97,6 @@ importlib-metadata==5.1.0
     # via
     #   jupyter-cache
     #   myst-nb
-    #   sphinx
-importlib-resources==5.10.1
-    # via jsonschema
 iniconfig==1.1.1
     # via pytest
 ipykernel==6.19.1
@@ -266,8 +257,6 @@ pip-tools==6.11.0
     # via sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 platformdirs==2.6.0
     # via
     #   black
@@ -288,6 +277,7 @@ psutil==5.9.4
     #   ipykernel
     #   memory-profiler
     #   pytest-monitor
+    #   sec-certs (./../pyproject.toml)
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
@@ -443,10 +433,8 @@ tomli==2.0.1
     #   pep517
     #   pytest
     #   setuptools-scm
-tornado==6.3.2
-    # via
-    #   ipykernel
-    #   jupyter-client
+tornado==6.3.3
+    # via setuptools-scm
 tqdm==4.64.1
     # via
     #   sec-certs (./../pyproject.toml)
@@ -476,12 +464,10 @@ types-urllib3==1.26.25.4
     # via types-requests
 typing-extensions==4.4.0
     # via
-    #   black
     #   mypy
     #   myst-nb
     #   myst-parser
     #   pydantic
-    #   pypdf
     #   setuptools-scm
 urllib3==1.26.13
     # via requests
@@ -502,9 +488,7 @@ wheel==0.38.4
 widgetsnbextension==4.0.4
     # via ipywidgets
 zipp==3.11.0
-    # via
-    #   importlib-metadata
-    #   importlib-resources
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 356ed82c..b0b5ca88 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -1,7 +1,3 @@
-appnope==0.1.3
-    # via
-    #   ipykernel
-    #   ipython
 asttokens==2.2.1
     # via stack-data
 attrs==22.1.0
@@ -58,8 +54,6 @@ html5lib==1.1
     # via sec-certs (./../pyproject.toml)
 idna==3.4
     # via requests
-importlib-resources==5.10.1
-    # via jsonschema
 ipykernel==6.19.1
     # via
     #   ipywidgets
@@ -161,8 +155,6 @@ pillow==9.3.0
     #   sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 platformdirs==2.6.0
     # via jupyter-core
 preshed==3.0.8
@@ -172,7 +164,9 @@ preshed==3.0.8
 prompt-toolkit==3.0.36
     # via ipython
 psutil==5.9.4
-    # via ipykernel
+    # via
+    #   ipykernel
+    #   sec-certs (./../pyproject.toml)
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
@@ -257,7 +251,7 @@ threadpoolctl==3.1.0
     # via scikit-learn
 tomli==2.0.1
     # via setuptools-scm
-tornado==6.3.2
+tornado==6.3.3
     # via
     #   ipykernel
     #   jupyter-client
@@ -281,7 +275,6 @@ typer==0.7.0
 typing-extensions==4.4.0
     # via
     #   pydantic
-    #   pypdf
     #   setuptools-scm
 urllib3==1.26.13
     # via requests
@@ -295,8 +288,6 @@ webencodings==0.5.1
     # via html5lib
 widgetsnbextension==4.0.4
     # via ipywidgets
-zipp==3.11.0
-    # via importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index b9ba98a4..2b92e95f 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -1,7 +1,3 @@
-appnope==0.1.3
-    # via
-    #   ipykernel
-    #   ipython
 asttokens==2.2.1
     # via stack-data
 attrs==22.1.0
@@ -56,8 +52,6 @@ distro==1.8.0
     # via tabula-py
 entrypoints==0.4
     # via jupyter-client
-exceptiongroup==1.0.4
-    # via pytest
 executing==1.2.0
     # via stack-data
 fonttools==4.38.0
@@ -66,8 +60,6 @@ html5lib==1.1
     # via sec-certs (./../pyproject.toml)
 idna==3.4
     # via requests
-importlib-resources==5.10.1
-    # via jsonschema
 iniconfig==1.1.1
     # via pytest
 ipykernel==6.19.1
@@ -172,8 +164,6 @@ pillow==9.3.0
     #   sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 platformdirs==2.6.0
     # via jupyter-core
 pluggy==1.0.0
@@ -185,7 +175,9 @@ preshed==3.0.8
 prompt-toolkit==3.0.36
     # via ipython
 psutil==5.9.4
-    # via ipykernel
+    # via
+    #   ipykernel
+    #   sec-certs (./../pyproject.toml)
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
@@ -279,7 +271,7 @@ tomli==2.0.1
     #   coverage
     #   pytest
     #   setuptools-scm
-tornado==6.3.2
+tornado==6.3.3
     # via
     #   ipykernel
     #   jupyter-client
@@ -303,7 +295,6 @@ typer==0.7.0
 typing-extensions==4.4.0
     # via
     #   pydantic
-    #   pypdf
     #   setuptools-scm
 urllib3==1.26.13
     # via requests
@@ -317,8 +308,6 @@ webencodings==0.5.1
     # via html5lib
 widgetsnbextension==4.0.4
     # via ipywidgets
-zipp==3.11.0
-    # via importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index 0db196c9..16c95e84 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -33,6 +33,7 @@ from sec_certs.sample.protection_profile import ProtectionProfile
 from sec_certs.serialization.json import ComplexSerializableType, serialize
 from sec_certs.utils import helpers
 from sec_certs.utils import parallel_processing as cert_processing
+from sec_certs.utils.profiling import staged
 
 
 @dataclass
@@ -270,6 +271,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         helpers.download_parallel(csv_urls, csv_paths)
 
     @serialize
+    @staged(logger, "Downloading and processing CSV and HTML files of certificates.")
     def get_certs_from_web(
         self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, get_archived: bool = True
     ) -> None:
@@ -520,12 +522,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._download_reports(fresh)
         self._download_targets(fresh)
 
+    @staged(logger, "Downloading PDFs of CC certification reports.")
     def _download_reports(self, fresh: bool = True) -> None:
         self.reports_pdf_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh) and x.report_link]
 
-        if fresh:
-            logger.info("Downloading PDFs of CC certification reports.")
         if not fresh and certs_to_process:
             logger.info(
                 f"Downloading {len(certs_to_process)} PDFs of CC certification reports for which previous download failed."
@@ -537,12 +538,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Downloading PDFs of CC certification reports",
         )
 
+    @staged(logger, "Downloading PDFs of CC security targets.")
     def _download_targets(self, fresh: bool = True) -> None:
         self.targets_pdf_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh)]
 
-        if fresh:
-            logger.info("Downloading PDFs of CC security targets.")
         if not fresh and certs_to_process:
             logger.info(
                 f"Downloading {len(certs_to_process)} PDFs of CC security targets for which previous download failed.."
@@ -554,12 +554,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Downloading PDFs of CC security targets",
         )
 
+    @staged(logger, "Converting PDFs of certification reports to txt.")
     def _convert_reports_to_txt(self, fresh: bool = True) -> None:
         self.reports_txt_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.report_is_ok_to_convert(fresh)]
 
-        if fresh:
-            logger.info("Converting PDFs of certification reports to txt.")
         if not fresh and certs_to_process:
             logger.info(
                 f"Converting {len(certs_to_process)} PDFs of certification reports to txt for which previous conversion failed."
@@ -571,6 +570,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Converting PDFs of certification reports to txt",
         )
 
+    @staged(logger, "Converting PDFs of security targets to txt.")
     def _convert_targets_to_txt(self, fresh: bool = True) -> None:
         self.targets_txt_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.st_is_ok_to_convert(fresh)]
@@ -592,8 +592,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._convert_reports_to_txt(fresh)
         self._convert_targets_to_txt(fresh)
 
+    @staged(logger, "Extracting report metadata")
     def _extract_report_metadata(self) -> None:
-        logger.info("Extracting report metadata")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_metadata,
@@ -603,8 +603,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    def _extract_targets_metadata(self) -> None:
-        logger.info("Extracting target metadata")
+    @staged(logger, "Extracting target metadata")
+    def _extract_target_metadata(self) -> None:
         certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_metadata,
@@ -616,10 +616,10 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     def _extract_pdf_metadata(self) -> None:
         self._extract_report_metadata()
-        self._extract_targets_metadata()
+        self._extract_target_metadata()
 
+    @staged(logger, "Extracting report frontpages")
     def _extract_report_frontpage(self) -> None:
-        logger.info("Extracting report frontpages")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_frontpage,
@@ -629,8 +629,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    def _extract_targets_frontpage(self) -> None:
-        logger.info("Extracting target frontpages")
+    @staged(logger, "Extracting target frontpages")
+    def _extract_target_frontpage(self) -> None:
         certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_frontpage,
@@ -642,10 +642,10 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     def _extract_pdf_frontpage(self) -> None:
         self._extract_report_frontpage()
-        self._extract_targets_frontpage()
+        self._extract_target_frontpage()
 
+    @staged(logger, "Extracting report keywords")
     def _extract_report_keywords(self) -> None:
-        logger.info("Extracting report keywords")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_keywords,
@@ -655,8 +655,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    def _extract_targets_keywords(self) -> None:
-        logger.info("Extracting target keywords")
+    @staged(logger, "Extracting target keywords")
+    def _extract_target_keywords(self) -> None:
         certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_keywords,
@@ -668,7 +668,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     def _extract_pdf_keywords(self) -> None:
         self._extract_report_keywords()
-        self._extract_targets_keywords()
+        self._extract_target_keywords()
 
     def extract_data(self) -> None:
         logger.info("Extracting various data from certification artifacts")
@@ -676,19 +676,19 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._extract_pdf_frontpage()
         self._extract_pdf_keywords()
 
+    @staged(logger, "Computing heuristics: Deriving information about laboratories involved in certification.")
     def _compute_cert_labs(self) -> None:
-        logger.info("Computing heuristics: Deriving information about laboratories involved in certification.")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         for cert in certs_to_process:
             cert.compute_heuristics_cert_lab()
 
+    @staged(logger, "Computing heuristics: Deriving information about certificate ids from artifacts.")
     def _compute_normalized_cert_ids(self) -> None:
-        logger.info("Computing heuristics: Deriving information about certificate ids from artifacts.")
         for cert in self:
             cert.compute_heuristics_cert_id()
 
+    @staged(logger, "Computing heuristics: Transitive vulnerabilities in referenc(ed/ing) certificates.")
     def _compute_transitive_vulnerabilities(self):
-        logger.info("omputing heuristics: computing transitive vulnerabilities in referenc(ed/ing) certificates.")
         transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: cert.heuristics.cert_id)
         transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.report_references)
 
@@ -698,9 +698,9 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves
             self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves
 
+    @staged(logger, "Computing heuristics: Matching scheme data.")
     def _compute_scheme_data(self):
         if self.auxiliary_datasets.scheme_dset:
-            print("here")
             for scheme in self.auxiliary_datasets.scheme_dset:
                 if certified := scheme.lists.get(EntryType.Certified):
                     certs = [cert for cert in self if cert.status == "active"]
@@ -713,6 +713,12 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
                     for dgst, match in matches.items():
                         self[dgst].heuristics.scheme_data = match
 
+    @staged(logger, "Computing heuristics: SARs")
+    def _compute_sars(self) -> None:
+        transformer = SARTransformer().fit(self.certs.values())
+        for cert in self:
+            cert.heuristics.extracted_sars = transformer.transform_single_cert(cert)
+
     def _compute_heuristics(self) -> None:
         self._compute_normalized_cert_ids()
         super()._compute_heuristics()
@@ -720,12 +726,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._compute_cert_labs()
         self._compute_sars()
 
-    def _compute_sars(self) -> None:
-        logger.info("Computing heuristics: Computing SARs")
-        transformer = SARTransformer().fit(self.certs.values())
-        for cert in self:
-            cert.heuristics.extracted_sars = transformer.transform_single_cert(cert)
-
+    @staged(logger, "Computing heuristics: references between certificates.")
     def _compute_references(self) -> None:
         def ref_lookup(kw_attr):
             def func(cert):
@@ -744,7 +745,6 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
             return func
 
-        logger.info("omputing heuristics: references between certificates.")
         for ref_source in ("report", "st"):
             kw_source = f"{ref_source}_keywords"
             dep_attr = f"{ref_source}_references"
@@ -768,6 +768,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             to_download=download_fresh, only_schemes={cert.scheme for cert in self}
         )
 
+    @staged(logger, "Processing protection profiles.")
     def process_protection_profiles(
         self, to_download: bool = True, keep_metadata: bool = True
     ) -> ProtectionProfileDataset:
@@ -779,7 +780,6 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         :param bool keep_metadata: If json related to the PP dataset should be kept on drive, defaults to True
         :raises RuntimeError: When building of PPDataset fails
         """
-        logger.info("Processing protection profiles.")
 
         self.auxiliary_datasets_dir.mkdir(parents=True, exist_ok=True)
 
@@ -798,13 +798,12 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
         return pp_dataset
 
+    @staged(logger, "Processing maintenace updates.")
     def process_maintenance_updates(self, to_download: bool = True) -> CCDatasetMaintenanceUpdates:
         """
         Downloads or loads from json a dataset of maintenance updates. Runs analysis on that dataset if it's not completed.
         :return CCDatasetMaintenanceUpdates: the resulting dataset of maintenance updates
         """
-
-        logger.info("Processing maintenace updates")
         self.mu_dataset_dir.mkdir(parents=True, exist_ok=True)
 
         if to_download or not self.mu_dataset_path.exists():
@@ -827,12 +826,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
         return update_dset
 
+    @staged(logger, "Processing CC scheme dataset.")
     def process_schemes(self, to_download: bool = True, only_schemes: set[str] | None = None) -> CCSchemeDataset:
         """
         Downloads or loads from json a dataset of CC scheme data.
         """
-        logger.info("Processing CC schemes")
-
         self.auxiliary_datasets_dir.mkdir(parents=True, exist_ok=True)
 
         if to_download or not self.scheme_dataset_path.exists():
diff --git a/src/sec_certs/dataset/dataset.py b/src/sec_certs/dataset/dataset.py
index b544294e..6466630f 100644
--- a/src/sec_certs/dataset/dataset.py
+++ b/src/sec_certs/dataset/dataset.py
@@ -25,6 +25,7 @@ from sec_certs.sample.cpe import CPE
 from sec_certs.serialization.json import ComplexSerializableType, get_class_fullname, serialize
 from sec_certs.utils import helpers
 from sec_certs.utils.nvd_dataset_builder import CpeMatchNvdDatasetBuilder, CpeNvdDatasetBuilder, CveNvdDatasetBuilder
+from sec_certs.utils.profiling import staged
 from sec_certs.utils.tqdm import tqdm
 
 logger = logging.getLogger(__name__)
@@ -297,8 +298,6 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
 
         logger.info("Converting all PDFs to txt")
         self._convert_all_pdfs_body(fresh)
-        if fresh:
-            self._convert_all_pdfs_body(False)
 
         self.state.pdfs_converted = True
 
@@ -350,6 +349,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
     def _compute_transitive_vulnerabilities(self) -> None:
         raise NotImplementedError("Not meant to be implemented by the base class.")
 
+    @staged(logger, "Processing CPEDataset.")
     def _prepare_cpe_dataset(self, download_fresh: bool = False) -> CPEDataset:
         if not self.auxiliary_datasets_dir.exists():
             self.auxiliary_datasets_dir.mkdir(parents=True)
@@ -373,6 +373,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
 
         return cpe_dataset
 
+    @staged(logger, "Processing CVEDataset.")
     def _prepare_cve_dataset(self, download_fresh: bool = False) -> CVEDataset:
         if not self.auxiliary_datasets_dir.exists():
             logger.info("Loading CVEDataset from json.")
@@ -397,6 +398,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
 
         return cve_dataset
 
+    @staged(logger, "Processing CPE match dict.")
     def _prepare_cpe_match_dict(self, download_fresh: bool = False) -> dict:
         if self.cpe_match_json_path.exists():
             logger.info("Preparing CPE Match feed from json.")
@@ -435,6 +437,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         return cpe_match_dict
 
     @serialize
+    @staged(logger, "Computing heuristics: Finding CPE matches for certificates")
     def compute_cpe_heuristics(self) -> CPEClassifier:
         """
         Computes matching CPEs for the certificates.
@@ -467,7 +470,6 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
                 return False
             return True
 
-        logger.info("Computing heuristics: Finding CPE matches for certificates")
         if not self.auxiliary_datasets.cpe_dset:
             self.auxiliary_datasets.cpe_dset = self._prepare_cpe_dataset()
 
@@ -576,11 +578,11 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         return set(itertools.chain.from_iterable(cpe_matches))
 
     @serialize
+    @staged(logger, "Computing heuristics: CVEs in certificates.")
     def compute_related_cves(self) -> None:
         """
         Computes CVEs for the certificates, given their CPE matches.
         """
-        logger.info("Computing heuristics: CVEs in certificates.")
 
         if not self.auxiliary_datasets.cpe_dset:
             self.auxiliary_datasets.cpe_dset = self._prepare_cpe_dataset()
diff --git a/src/sec_certs/dataset/fips.py b/src/sec_certs/dataset/fips.py
index 24536b9b..a3d24b15 100644
--- a/src/sec_certs/dataset/fips.py
+++ b/src/sec_certs/dataset/fips.py
@@ -24,6 +24,7 @@ from sec_certs.serialization.json import ComplexSerializableType, serialize
 from sec_certs.utils import helpers
 from sec_certs.utils import parallel_processing as cert_processing
 from sec_certs.utils.helpers import fips_dgst
+from sec_certs.utils.profiling import staged
 
 logger = logging.getLogger(__name__)
 
@@ -230,13 +231,13 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
             cert.set_local_paths(self.policies_pdf_dir, self.policies_txt_dir, self.module_dir)
 
     @serialize
+    @staged(logger, "Downloading and processing certificates.")
     def get_certs_from_web(self, to_download: bool = True, keep_metadata: bool = True) -> None:
         self.web_dir.mkdir(parents=True, exist_ok=True)
 
         if to_download:
             self._download_html_resources()
 
-        logger.info("Adding unprocessed FIPS certificates into FIPSDataset.")
         self.certs = {x.dgst: x for x in self._get_all_certs_from_html_sources()}
         logger.info(f"The dataset now contains {len(self)} certificates.")
 
@@ -251,8 +252,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
         super().process_auxiliary_datasets(download_fresh)
         self.auxiliary_datasets.algorithm_dset = self._prepare_algorithm_dataset(download_fresh)
 
+    @staged(logger, "Processing FIPSAlgorithm dataset.")
     def _prepare_algorithm_dataset(self, download_fresh_algs: bool = False) -> FIPSAlgorithmDataset:
-        logger.info("Preparing FIPSAlgorithm dataset.")
         if not self.algorithm_dataset_path.exists() or download_fresh_algs:
             alg_dset = FIPSAlgorithmDataset.from_web(self.algorithm_dataset_path)
             alg_dset.to_json()
@@ -261,8 +262,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
 
         return alg_dset
 
+    @staged(logger, "Extracting Algorithms from policy tables")
     def _extract_algorithms_from_policy_tables(self):
-        logger.info("Extracting Algorithms from policy tables")
         certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()]
         cert_processing.process_parallel(
             FIPSCertificate.get_algorithms_from_policy_tables,
@@ -271,8 +272,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
             progress_bar_desc="Extracting Algorithms from policy tables",
         )
 
+    @staged(logger, "Extracting security policy metadata from the pdfs")
     def _extract_policy_pdf_metadata(self) -> None:
-        logger.info("Extracting security policy metadata from the pdfs")
         certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             FIPSCertificate.extract_policy_pdf_metadata,
@@ -282,8 +283,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
         )
         self.update_with_certs(processed_certs)
 
+    @staged(logger, "Computing heuristics: Transitive vulnerabilities in referenc(ed/ing) certificates.")
     def _compute_transitive_vulnerabilities(self) -> None:
-        logger.info("Computing heuristics: Computing transitive vulnerabilities in referenc(ed/ing) certificates.")
         transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: str(cert.cert_id))
         transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.policy_processed_references)
 
@@ -292,20 +293,16 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
             self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves
             self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves
 
-    def _prune_reference_candidates(self) -> None:
-        for cert in self:
-            cert.prune_referenced_cert_ids()
-
+    @staged(logger, "Computing heuristics: references between certificates.")
+    def _compute_references(self, keep_unknowns: bool = False) -> None:
         # Previously, a following procedure was used to prune reference_candidates:
         #   - A set of algorithms was obtained via self.auxiliary_datasets.algorithm_dset.get_algorithms_by_id(reference_candidate)
         #   - If any of these algorithms had the same vendor as the reference_candidate, the candidate was rejected
         #   - The rationale is that if an ID appears in a certificate s.t. an algorithm with the same ID was produced by the same vendor, the reference likely refers to alg.
         #   - Such reference should then be discarded.
         #   - We are uncertain of the effectivity of such measure, disabling it for now.
-
-    def _compute_references(self, keep_unknowns: bool = False) -> None:
-        logger.info("Computing heuristics: Recovering references between certificates")
-        self._prune_reference_candidates()
+        for cert in self:
+            cert.prune_referenced_cert_ids()
 
         policy_reference_finder = ReferenceFinder()
         policy_reference_finder.fit(
diff --git a/src/sec_certs/model/matching.py b/src/sec_certs/model/matching.py
index b298e316..7e48f878 100644
--- a/src/sec_certs/model/matching.py
+++ b/src/sec_certs/model/matching.py
@@ -49,6 +49,8 @@ class AbstractMatcher(Generic[CertSubType], ABC):
             if score < threshold:
                 break
             # Match cert dgst to entry
+            matched_is.add(i)
+            matched_js.add(j)
             cert = certs[i]
             entry = matchers[j].entry
             results[cert.dgst] = entry
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 3e28dfc8..d6e49718 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -115,6 +115,14 @@ class CertificateId:
         cert_num = int(new_cert_id.split("-")[1])
         return f"SERTIT-{cert_num:03}"
 
+    def _canonical_nl(self):
+        new_cert_id = self.clean
+        if new_cert_id.startswith("CC-"):
+            new_cert_id = f"NSCIB-{new_cert_id}"
+        if not new_cert_id.endswith("-CR"):
+            new_cert_id = f"{new_cert_id}-CR"
+        return new_cert_id
+
     @property
     def clean(self) -> str:
         """
@@ -139,6 +147,7 @@ class CertificateId:
             "CA": self._canonical_ca,
             "JP": self._canonical_jp,
             "NO": self._canonical_no,
+            "NL": self._canonical_nl,
         }
 
         if self.scheme in schemes:
diff --git a/src/sec_certs/utils/profiling.py b/src/sec_certs/utils/profiling.py
new file mode 100644
index 00000000..7da67070
--- /dev/null
+++ b/src/sec_certs/utils/profiling.py
@@ -0,0 +1,45 @@
+import gc
+from contextlib import contextmanager
+from datetime import datetime
+from functools import wraps
+from logging import Logger
+
+import psutil
+
+
+@contextmanager
+def log_stage(logger: Logger, msg: str, collect_garbage: bool = False):
+    """Contextmanager that logs a message to the logger when it is entered and exited.
+    The message has debug information about memory use. Optionally, it can
+    run garbage collection when exiting.
+    """
+    meminfo = psutil.Process().memory_full_info()
+    logger.info(f">> Starting >> {msg}")
+    logger.debug(str(meminfo))
+    start_time = datetime.now()
+
+    try:
+        yield
+    finally:
+        end_time = datetime.now()
+        duration = end_time - start_time
+        meminfo = psutil.Process().memory_full_info()
+        logger.info(f"<< Finished << {msg} ({duration})")
+        logger.debug(str(meminfo))
+
+        if collect_garbage:
+            gc.collect()
+
+
+def staged(logger: Logger, log_message: str, collect_garbage: bool = False):
+    """Like log_stage but a decorator."""
+
+    def deco(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            with log_stage(logger, log_message, collect_garbage):
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return deco