merge fresh main

19 files changed, 248 insertions, 90 deletions

diff --git a/src/sec_certs/configuration.py b/src/sec_certs/configuration.py
index 3e49ecad..d7fb734a 100644
--- a/src/sec_certs/configuration.py
+++ b/src/sec_certs/configuration.py
@@ -18,7 +18,7 @@ class Configuration(BaseSettings):
         env_prefix = "seccerts_"
 
     log_filepath: Path = Field(
-        "./cert_processing_log.txt",
+        "./cert_processing_log.log",
         description="Path to the file, relative to working directory, where the log will be stored.",
     )
     always_false_positive_fips_cert_id_threshold: int = Field(
diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
index 773a5720..01649bb7 100644
--- a/src/sec_certs/constants.py
+++ b/src/sec_certs/constants.py
@@ -76,10 +76,12 @@ GARBAGE_ALPHA_CHARS_THRESHOLD = 0.5
 CC_AUSTRALIA_BASE_URL = "https://www.cyber.gov.au"
 CC_AUSTRALIA_INEVAL_URL = (
     CC_AUSTRALIA_BASE_URL
-    + "/resources-business-and-government/assessment-and-evaluation-programs/australian-information-security-evaluation-program-aisep"
+    + "/resources-business-and-government/assessment-and-evaluation-programs/australian-information-security-evaluation-program"
 )
-CC_CANADA_CERTIFIED_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/certified-products"
-CC_CANADA_INEVAL_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/products-evaluation"
+CC_CANADA_BASE_URL = "https://www.cyber.gc.ca"
+CC_CANADA_API_URL = CC_CANADA_BASE_URL + "/api/cccs/page/v1/get"
+CC_CANADA_CERTIFIED_URL = "/en/tools-services/common-criteria/certified-products"
+CC_CANADA_INEVAL_URL = "/en/tools-services/common-criteria/products-evaluation"
 CC_ANSSI_BASE_URL = "https://www.ssi.gouv.fr"
 CC_ANSSI_CERTIFIED_URL = CC_ANSSI_BASE_URL + "/en/products/certified-products/"
 CC_BSI_BASE_URL = "https://www.bsi.bund.de/"
diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index afde8bd2..9e382ca7 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -35,6 +35,7 @@ from sec_certs.sample.protection_profile import ProtectionProfile
 from sec_certs.serialization.json import ComplexSerializableType, serialize
 from sec_certs.utils import helpers
 from sec_certs.utils import parallel_processing as cert_processing
+from sec_certs.utils.profiling import staged
 
 
 @dataclass
@@ -276,6 +277,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         helpers.download_parallel(csv_urls, csv_paths)
 
     @serialize
+    @staged(logger, "Downloading and processing CSV and HTML files of certificates.")
     def get_certs_from_web(
         self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, get_archived: bool = True
     ) -> None:
@@ -526,12 +528,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._download_reports(fresh)
         self._download_targets(fresh)
 
+    @staged(logger, "Downloading PDFs of CC certification reports.")
     def _download_reports(self, fresh: bool = True) -> None:
         self.reports_pdf_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh) and x.report_link]
 
-        if fresh:
-            logger.info("Downloading PDFs of CC certification reports.")
         if not fresh and certs_to_process:
             logger.info(
                 f"Downloading {len(certs_to_process)} PDFs of CC certification reports for which previous download failed."
@@ -543,12 +544,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Downloading PDFs of CC certification reports",
         )
 
+    @staged(logger, "Downloading PDFs of CC security targets.")
     def _download_targets(self, fresh: bool = True) -> None:
         self.targets_pdf_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh)]
 
-        if fresh:
-            logger.info("Downloading PDFs of CC security targets.")
         if not fresh and certs_to_process:
             logger.info(
                 f"Downloading {len(certs_to_process)} PDFs of CC security targets for which previous download failed.."
@@ -560,12 +560,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Downloading PDFs of CC security targets",
         )
 
+    @staged(logger, "Converting PDFs of certification reports to txt.")
     def _convert_reports_to_txt(self, fresh: bool = True) -> None:
         self.reports_txt_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.report_is_ok_to_convert(fresh)]
 
-        if fresh:
-            logger.info("Converting PDFs of certification reports to txt.")
         if not fresh and certs_to_process:
             logger.info(
                 f"Converting {len(certs_to_process)} PDFs of certification reports to txt for which previous conversion failed."
@@ -577,6 +576,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Converting PDFs of certification reports to txt",
         )
 
+    @staged(logger, "Converting PDFs of security targets to txt.")
     def _convert_targets_to_txt(self, fresh: bool = True) -> None:
         self.targets_txt_dir.mkdir(parents=True, exist_ok=True)
         certs_to_process = [x for x in self if x.state.st_is_ok_to_convert(fresh)]
@@ -598,8 +598,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._convert_reports_to_txt(fresh)
         self._convert_targets_to_txt(fresh)
 
+    @staged(logger, "Extracting report metadata")
     def _extract_report_metadata(self) -> None:
-        logger.info("Extracting report metadata")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_metadata,
@@ -609,8 +609,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    def _extract_targets_metadata(self) -> None:
-        logger.info("Extracting target metadata")
+    @staged(logger, "Extracting target metadata")
+    def _extract_target_metadata(self) -> None:
         certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_metadata,
@@ -622,10 +622,10 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     def _extract_pdf_metadata(self) -> None:
         self._extract_report_metadata()
-        self._extract_targets_metadata()
+        self._extract_target_metadata()
 
+    @staged(logger, "Extracting report frontpages")
     def _extract_report_frontpage(self) -> None:
-        logger.info("Extracting report frontpages")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_frontpage,
@@ -635,8 +635,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    def _extract_targets_frontpage(self) -> None:
-        logger.info("Extracting target frontpages")
+    @staged(logger, "Extracting target frontpages")
+    def _extract_target_frontpage(self) -> None:
         certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_frontpage,
@@ -648,10 +648,10 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     def _extract_pdf_frontpage(self) -> None:
         self._extract_report_frontpage()
-        self._extract_targets_frontpage()
+        self._extract_target_frontpage()
 
+    @staged(logger, "Extracting report keywords")
     def _extract_report_keywords(self) -> None:
-        logger.info("Extracting report keywords")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_keywords,
@@ -661,8 +661,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    def _extract_targets_keywords(self) -> None:
-        logger.info("Extracting target keywords")
+    @staged(logger, "Extracting target keywords")
+    def _extract_target_keywords(self) -> None:
         certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_keywords,
@@ -674,7 +674,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     def _extract_pdf_keywords(self) -> None:
         self._extract_report_keywords()
-        self._extract_targets_keywords()
+        self._extract_target_keywords()
 
     def extract_data(self) -> None:
         logger.info("Extracting various data from certification artifacts")
@@ -682,19 +682,19 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._extract_pdf_frontpage()
         self._extract_pdf_keywords()
 
+    @staged(logger, "Computing heuristics: Deriving information about laboratories involved in certification.")
     def _compute_cert_labs(self) -> None:
-        logger.info("Computing heuristics: Deriving information about laboratories involved in certification.")
         certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
         for cert in certs_to_process:
             cert.compute_heuristics_cert_lab()
 
+    @staged(logger, "Computing heuristics: Deriving information about certificate ids from artifacts.")
     def _compute_normalized_cert_ids(self) -> None:
-        logger.info("Computing heuristics: Deriving information about certificate ids from artifacts.")
         for cert in self:
             cert.compute_heuristics_cert_id()
 
+    @staged(logger, "Computing heuristics: Transitive vulnerabilities in referenc(ed/ing) certificates.")
     def _compute_transitive_vulnerabilities(self):
-        logger.info("omputing heuristics: computing transitive vulnerabilities in referenc(ed/ing) certificates.")
         transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: cert.heuristics.cert_id)
         transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.report_references)
 
@@ -704,9 +704,9 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves
             self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves
 
+    @staged(logger, "Computing heuristics: Matching scheme data.")
     def _compute_scheme_data(self):
         if self.auxiliary_datasets.scheme_dset:
-            print("here")
             for scheme in self.auxiliary_datasets.scheme_dset:
                 if certified := scheme.lists.get(EntryType.Certified):
                     certs = [cert for cert in self if cert.status == "active"]
@@ -719,6 +719,12 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
                     for dgst, match in matches.items():
                         self[dgst].heuristics.scheme_data = match
 
+    @staged(logger, "Computing heuristics: SARs")
+    def _compute_sars(self) -> None:
+        transformer = SARTransformer().fit(self.certs.values())
+        for cert in self:
+            cert.heuristics.extracted_sars = transformer.transform_single_cert(cert)
+
     def _compute_heuristics(self) -> None:
         self._compute_normalized_cert_ids()
         super()._compute_heuristics()
@@ -726,12 +732,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         self._compute_cert_labs()
         self._compute_sars()
 
-    def _compute_sars(self) -> None:
-        logger.info("Computing heuristics: Computing SARs")
-        transformer = SARTransformer().fit(self.certs.values())
-        for cert in self:
-            cert.heuristics.extracted_sars = transformer.transform_single_cert(cert)
-
+    @staged(logger, "Computing heuristics: references between certificates.")
     def _compute_references(self) -> None:
         def ref_lookup(kw_attr):
             def func(cert):
@@ -750,7 +751,6 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
             return func
 
-        logger.info("omputing heuristics: references between certificates.")
         for ref_source in ("report", "st"):
             kw_source = f"{ref_source}_keywords"
             dep_attr = f"{ref_source}_references"
@@ -774,6 +774,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             to_download=download_fresh, only_schemes={cert.scheme for cert in self}
         )
 
+    @staged(logger, "Processing protection profiles.")
     def process_protection_profiles(
         self, to_download: bool = True, keep_metadata: bool = True
     ) -> ProtectionProfileDataset:
@@ -785,7 +786,6 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         :param bool keep_metadata: If json related to the PP dataset should be kept on drive, defaults to True
         :raises RuntimeError: When building of PPDataset fails
         """
-        logger.info("Processing protection profiles.")
 
         self.auxiliary_datasets_dir.mkdir(parents=True, exist_ok=True)
 
@@ -804,13 +804,12 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
         return pp_dataset
 
+    @staged(logger, "Processing maintenace updates.")
     def process_maintenance_updates(self, to_download: bool = True) -> CCDatasetMaintenanceUpdates:
         """
         Downloads or loads from json a dataset of maintenance updates. Runs analysis on that dataset if it's not completed.
         :return CCDatasetMaintenanceUpdates: the resulting dataset of maintenance updates
         """
-
-        logger.info("Processing maintenace updates")
         self.mu_dataset_dir.mkdir(parents=True, exist_ok=True)
 
         if to_download or not self.mu_dataset_path.exists():
@@ -833,12 +832,11 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
         return update_dset
 
+    @staged(logger, "Processing CC scheme dataset.")
     def process_schemes(self, to_download: bool = True, only_schemes: set[str] | None = None) -> CCSchemeDataset:
         """
         Downloads or loads from json a dataset of CC scheme data.
         """
-        logger.info("Processing CC schemes")
-
         self.auxiliary_datasets_dir.mkdir(parents=True, exist_ok=True)
 
         if to_download or not self.scheme_dataset_path.exists():
diff --git a/src/sec_certs/dataset/cc_scheme.py b/src/sec_certs/dataset/cc_scheme.py
index 0625b257..a2550014 100644
--- a/src/sec_certs/dataset/cc_scheme.py
+++ b/src/sec_certs/dataset/cc_scheme.py
@@ -54,5 +54,8 @@ class CCSchemeDataset(JSONPathDataset, ComplexSerializableType):
         for scheme, sources in CCScheme.methods.items():
             if only_schemes is not None and scheme not in only_schemes:
                 continue
-            schemes[scheme] = CCScheme.from_web(scheme, sources.keys())
+            try:
+                schemes[scheme] = CCScheme.from_web(scheme, sources.keys())
+            except Exception as e:
+                logger.warning(f"Could not download CC scheme: {scheme} due to error {e}.")
         return cls(schemes)
diff --git a/src/sec_certs/dataset/dataset.py b/src/sec_certs/dataset/dataset.py
index b544294e..454e86bd 100644
--- a/src/sec_certs/dataset/dataset.py
+++ b/src/sec_certs/dataset/dataset.py
@@ -25,6 +25,7 @@ from sec_certs.sample.cpe import CPE
 from sec_certs.serialization.json import ComplexSerializableType, get_class_fullname, serialize
 from sec_certs.utils import helpers
 from sec_certs.utils.nvd_dataset_builder import CpeMatchNvdDatasetBuilder, CpeNvdDatasetBuilder, CveNvdDatasetBuilder
+from sec_certs.utils.profiling import staged
 from sec_certs.utils.tqdm import tqdm
 
 logger = logging.getLogger(__name__)
@@ -297,8 +298,6 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
 
         logger.info("Converting all PDFs to txt")
         self._convert_all_pdfs_body(fresh)
-        if fresh:
-            self._convert_all_pdfs_body(False)
 
         self.state.pdfs_converted = True
 
@@ -350,6 +349,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
     def _compute_transitive_vulnerabilities(self) -> None:
         raise NotImplementedError("Not meant to be implemented by the base class.")
 
+    @staged(logger, "Processing CPEDataset.")
     def _prepare_cpe_dataset(self, download_fresh: bool = False) -> CPEDataset:
         if not self.auxiliary_datasets_dir.exists():
             self.auxiliary_datasets_dir.mkdir(parents=True)
@@ -373,6 +373,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
 
         return cpe_dataset
 
+    @staged(logger, "Processing CVEDataset.")
     def _prepare_cve_dataset(self, download_fresh: bool = False) -> CVEDataset:
         if not self.auxiliary_datasets_dir.exists():
             logger.info("Loading CVEDataset from json.")
@@ -397,6 +398,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
 
         return cve_dataset
 
+    @staged(logger, "Processing CPE match dict.")
     def _prepare_cpe_match_dict(self, download_fresh: bool = False) -> dict:
         if self.cpe_match_json_path.exists():
             logger.info("Preparing CPE Match feed from json.")
@@ -435,6 +437,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         return cpe_match_dict
 
     @serialize
+    @staged(logger, "Computing heuristics: Finding CPE matches for certificates")
     def compute_cpe_heuristics(self) -> CPEClassifier:
         """
         Computes matching CPEs for the certificates.
@@ -467,11 +470,14 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
                 return False
             return True
 
-        logger.info("Computing heuristics: Finding CPE matches for certificates")
         if not self.auxiliary_datasets.cpe_dset:
             self.auxiliary_datasets.cpe_dset = self._prepare_cpe_dataset()
 
         clf = CPEClassifier(config.cpe_matching_threshold, config.cpe_n_max_matches)
+
+        if self.auxiliary_datasets.cpe_dset is None:
+            raise ValueError("CPE dataset cannot be None")
+
         clf.fit([x for x in self.auxiliary_datasets.cpe_dset if filter_condition(x)])
 
         cert: CertSubType
@@ -496,7 +502,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
             candidates = [cpe_dset[x].title for x in cert.heuristics.cpe_matches]
             candidates += ["No good match"] * (config.cpe_n_max_matches - len(candidates))
             options = ["option_" + str(x) for x in range(1, config.cpe_n_max_matches)]
-            dct.update({o: c for o, c in zip(options, candidates)})
+            dct.update(dict(zip(options, candidates)))
             lst.append(dct)
 
         with Path(output_path).open("w") as handle:
@@ -576,11 +582,11 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         return set(itertools.chain.from_iterable(cpe_matches))
 
     @serialize
+    @staged(logger, "Computing heuristics: CVEs in certificates.")
     def compute_related_cves(self) -> None:
         """
         Computes CVEs for the certificates, given their CPE matches.
         """
-        logger.info("Computing heuristics: CVEs in certificates.")
 
         if not self.auxiliary_datasets.cpe_dset:
             self.auxiliary_datasets.cpe_dset = self._prepare_cpe_dataset()
@@ -588,6 +594,9 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         if not self.auxiliary_datasets.cve_dset:
             self.auxiliary_datasets.cve_dset = self._prepare_cve_dataset()
 
+        if self.auxiliary_datasets.cve_dset is None:
+            raise ValueError("CVE dataset cannot be None")
+
         if not self.auxiliary_datasets.cve_dset.look_up_dicts_built:
             cpe_match_dict = self._prepare_cpe_match_dict()
             all_cpes = self._get_all_cpes_in_dataset()
@@ -625,6 +634,6 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         Enriches the dataset with `certs`
         :param List[Certificate] certs: new certs to include into the dataset.
         """
-        if any([x not in self for x in certs]):
+        if any(x not in self for x in certs):
             logger.warning("Updating dataset with certificates outside of the dataset!")
         self.certs.update({x.dgst: x for x in certs})
diff --git a/src/sec_certs/dataset/fips.py b/src/sec_certs/dataset/fips.py
index 24536b9b..a3d24b15 100644
--- a/src/sec_certs/dataset/fips.py
+++ b/src/sec_certs/dataset/fips.py
@@ -24,6 +24,7 @@ from sec_certs.serialization.json import ComplexSerializableType, serialize
 from sec_certs.utils import helpers
 from sec_certs.utils import parallel_processing as cert_processing
 from sec_certs.utils.helpers import fips_dgst
+from sec_certs.utils.profiling import staged
 
 logger = logging.getLogger(__name__)
 
@@ -230,13 +231,13 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
             cert.set_local_paths(self.policies_pdf_dir, self.policies_txt_dir, self.module_dir)
 
     @serialize
+    @staged(logger, "Downloading and processing certificates.")
     def get_certs_from_web(self, to_download: bool = True, keep_metadata: bool = True) -> None:
         self.web_dir.mkdir(parents=True, exist_ok=True)
 
         if to_download:
             self._download_html_resources()
 
-        logger.info("Adding unprocessed FIPS certificates into FIPSDataset.")
         self.certs = {x.dgst: x for x in self._get_all_certs_from_html_sources()}
         logger.info(f"The dataset now contains {len(self)} certificates.")
 
@@ -251,8 +252,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
         super().process_auxiliary_datasets(download_fresh)
         self.auxiliary_datasets.algorithm_dset = self._prepare_algorithm_dataset(download_fresh)
 
+    @staged(logger, "Processing FIPSAlgorithm dataset.")
     def _prepare_algorithm_dataset(self, download_fresh_algs: bool = False) -> FIPSAlgorithmDataset:
-        logger.info("Preparing FIPSAlgorithm dataset.")
         if not self.algorithm_dataset_path.exists() or download_fresh_algs:
             alg_dset = FIPSAlgorithmDataset.from_web(self.algorithm_dataset_path)
             alg_dset.to_json()
@@ -261,8 +262,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
 
         return alg_dset
 
+    @staged(logger, "Extracting Algorithms from policy tables")
     def _extract_algorithms_from_policy_tables(self):
-        logger.info("Extracting Algorithms from policy tables")
         certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()]
         cert_processing.process_parallel(
             FIPSCertificate.get_algorithms_from_policy_tables,
@@ -271,8 +272,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
             progress_bar_desc="Extracting Algorithms from policy tables",
         )
 
+    @staged(logger, "Extracting security policy metadata from the pdfs")
     def _extract_policy_pdf_metadata(self) -> None:
-        logger.info("Extracting security policy metadata from the pdfs")
         certs_to_process = [x for x in self if x.state.policy_is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             FIPSCertificate.extract_policy_pdf_metadata,
@@ -282,8 +283,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
         )
         self.update_with_certs(processed_certs)
 
+    @staged(logger, "Computing heuristics: Transitive vulnerabilities in referenc(ed/ing) certificates.")
     def _compute_transitive_vulnerabilities(self) -> None:
-        logger.info("Computing heuristics: Computing transitive vulnerabilities in referenc(ed/ing) certificates.")
         transitive_cve_finder = TransitiveVulnerabilityFinder(lambda cert: str(cert.cert_id))
         transitive_cve_finder.fit(self.certs, lambda cert: cert.heuristics.policy_processed_references)
 
@@ -292,20 +293,16 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
             self.certs[dgst].heuristics.direct_transitive_cves = transitive_cve.direct_transitive_cves
             self.certs[dgst].heuristics.indirect_transitive_cves = transitive_cve.indirect_transitive_cves
 
-    def _prune_reference_candidates(self) -> None:
-        for cert in self:
-            cert.prune_referenced_cert_ids()
-
+    @staged(logger, "Computing heuristics: references between certificates.")
+    def _compute_references(self, keep_unknowns: bool = False) -> None:
         # Previously, a following procedure was used to prune reference_candidates:
         #   - A set of algorithms was obtained via self.auxiliary_datasets.algorithm_dset.get_algorithms_by_id(reference_candidate)
         #   - If any of these algorithms had the same vendor as the reference_candidate, the candidate was rejected
         #   - The rationale is that if an ID appears in a certificate s.t. an algorithm with the same ID was produced by the same vendor, the reference likely refers to alg.
         #   - Such reference should then be discarded.
         #   - We are uncertain of the effectivity of such measure, disabling it for now.
-
-    def _compute_references(self, keep_unknowns: bool = False) -> None:
-        logger.info("Computing heuristics: Recovering references between certificates")
-        self._prune_reference_candidates()
+        for cert in self:
+            cert.prune_referenced_cert_ids()
 
         policy_reference_finder = ReferenceFinder()
         policy_reference_finder.fit(
diff --git a/src/sec_certs/dataset/fips_algorithm.py b/src/sec_certs/dataset/fips_algorithm.py
index fbc8351a..f3f27c0b 100644
--- a/src/sec_certs/dataset/fips_algorithm.py
+++ b/src/sec_certs/dataset/fips_algorithm.py
@@ -82,7 +82,7 @@ class FIPSAlgorithmDataset(JSONPathDataset, ComplexSerializableType):
         if failed_tuples:
             failed_urls, failed_paths = zip(*failed_tuples)
             responses = helpers.download_parallel(failed_urls, failed_paths)
-            if any([x != constants.RESPONSE_OK for x in responses]):
+            if any(x != constants.RESPONSE_OK for x in responses):
                 raise ValueError("Failed to download the algorithms HTML data, the dataset won't be constructed.")
 
         return paths
diff --git a/src/sec_certs/model/matching.py b/src/sec_certs/model/matching.py
index b298e316..9c0591f9 100644
--- a/src/sec_certs/model/matching.py
+++ b/src/sec_certs/model/matching.py
@@ -28,7 +28,7 @@ class AbstractMatcher(Generic[CertSubType], ABC):
         )
 
     @staticmethod
-    def _match_certs(matchers: Sequence["AbstractMatcher"], certs: list[CertSubType], threshold: float):
+    def _match_certs(matchers: Sequence[AbstractMatcher], certs: list[CertSubType], threshold: float):
         scores: list[tuple[float, int, int]] = []
         matched_is: set[int] = set()
         matched_js: set[int] = set()
@@ -49,6 +49,8 @@ class AbstractMatcher(Generic[CertSubType], ABC):
             if score < threshold:
                 break
             # Match cert dgst to entry
+            matched_is.add(i)
+            matched_js.add(j)
             cert = certs[i]
             entry = matchers[j].entry
             results[cert.dgst] = entry
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 8072df99..32c875c8 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -247,7 +247,7 @@ class CCCertificate(
         st_filename: str | None = field(default=None)
 
         def __bool__(self) -> bool:
-            return any([x is not None for x in vars(self)])
+            return any(x is not None for x in vars(self))
 
         @property
         def bsi_data(self) -> dict[str, Any] | None:
@@ -616,13 +616,12 @@ class CCCertificate(
             )
 
         for att, val in vars(self).items():
-            if not val:
-                setattr(self, att, getattr(other, att))
-            elif other_source == "html" and att == "protection_profiles":
-                setattr(self, att, getattr(other, att))
-            elif other_source == "html" and att == "maintenance_updates":
-                setattr(self, att, getattr(other, att))
-            elif att == "state":
+            if (
+                (not val)
+                or (other_source == "html" and att == "protection_profiles")
+                or (other_source == "html" and att == "maintenance_updates")
+                or (att == "state")
+            ):
                 setattr(self, att, getattr(other, att))
             else:
                 if getattr(self, att) != getattr(other, att):
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 1342cab8..254c8e05 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -122,6 +122,14 @@ class CertificateId:
         cert_num = int(new_cert_id.split("-")[1])
         return f"SERTIT-{cert_num:03}"
 
+    def _canonical_nl(self):
+        new_cert_id = self.clean
+        if new_cert_id.startswith("CC-"):
+            new_cert_id = f"NSCIB-{new_cert_id}"
+        if not new_cert_id.endswith("-CR"):
+            new_cert_id = f"{new_cert_id}-CR"
+        return new_cert_id
+
     @property
     def clean(self) -> str:
         """
@@ -146,6 +154,7 @@ class CertificateId:
             "CA": self._canonical_ca,
             "JP": self._canonical_jp,
             "NO": self._canonical_no,
+            "NL": self._canonical_nl,
         }
 
         if self.scheme in schemes:
diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py
index 12fb1fd4..bc0c0792 100644
--- a/src/sec_certs/sample/cc_scheme.py
+++ b/src/sec_certs/sample/cc_scheme.py
@@ -107,7 +107,7 @@ def get_australia_in_evaluation(enhanced: bool = True) -> list[dict[str, Any]]:
         if enhanced:
             e: dict[str, Any] = {}
             cert_page = _get_page(cert["url"])
-            article = cert_page.find("article", attrs={"role": "article"})
+            article = cert_page.find("article")
             blocks = article.find("div").find_all("div", class_="flex", recursive=False)
             for h2 in blocks[0].find_all("h2"):
                 val = sns(h2.find_next_sibling("span").text)
@@ -153,7 +153,9 @@ def get_canada_certified() -> list[dict[str, Any]]:
 
     :return: The entries.
     """
-    soup = _get_page(constants.CC_CANADA_CERTIFIED_URL)
+    resp = _get(constants.CC_CANADA_API_URL + f"?lang=en&url={constants.CC_CANADA_CERTIFIED_URL}", None)
+    html_data = resp.json()["response"]["page"]["body"][0]
+    soup = BeautifulSoup(html_data, "html5lib")
     tbody = soup.find("table").find("tbody")
     results = []
     for tr in tqdm(tbody.find_all("tr"), desc="Get CA scheme certified."):
@@ -176,7 +178,9 @@ def get_canada_in_evaluation() -> list[dict[str, Any]]:
 
     :return: The entries.
     """
-    soup = _get_page(constants.CC_CANADA_INEVAL_URL)
+    resp = _get(constants.CC_CANADA_API_URL + f"?lang=en&url={constants.CC_CANADA_INEVAL_URL}", None)
+    html_data = resp.json()["response"]["page"]["body"][0]
+    soup = BeautifulSoup(html_data, "html5lib")
     tbody = soup.find("table").find("tbody")
     results = []
     for tr in tqdm(tbody.find_all("tr"), desc="Get CA scheme in evaluation."):
diff --git a/src/sec_certs/sample/fips.py b/src/sec_certs/sample/fips.py
index d351c3ce..e0bcb74a 100644
--- a/src/sec_certs/sample/fips.py
+++ b/src/sec_certs/sample/fips.py
@@ -64,7 +64,7 @@ class FIPSHTMLParser:
         )
         entries = [(FIPSHTMLParser.normalize_string(key.text), entry) for key, entry in entries]
         entries = [parse_single_detail_entry(*x) for x in entries if x[0] in DETAILS_KEY_NORMALIZATION_DICT]
-        entries = {x: y for x, y in entries}
+        entries = dict(entries)
 
         if "caveat" in entries:
             entries["mentioned_certs"] = FIPSHTMLParser.get_mentioned_certs_from_caveat(entries["caveat"])
@@ -83,15 +83,13 @@ class FIPSHTMLParser:
     def _build_vendor_dict(vendor_div: Tag) -> dict[str, Any]:
         if not (link := vendor_div.find("a")):
             return {"vendor_url": None, "vendor": list(vendor_div.find("div", "panel-body").children)[0].strip()}
-        else:
-            return {"vendor_url": link.get("href"), "vendor": link.text.strip()}
+        return {"vendor_url": link.get("href"), "vendor": link.text.strip()}
 
     @staticmethod
     def _build_related_files_dict(related_files_div: Tag) -> dict[str, Any]:
         if cert_link := [x for x in related_files_div.find_all("a") if "Certificate" in x.text]:
             return {"certificate_pdf_url": constants.FIPS_BASE_URL + cert_link[0].get("href")}
-        else:
-            return {"certificate_pdf_url": None}
+        return {"certificate_pdf_url": None}
 
     @staticmethod
     def _build_validation_history_dict(validation_history_div: Tag) -> dict[str, Any]:
@@ -433,8 +431,7 @@ class FIPSCertificate(
                 fips_certlike = self.keywords["fips_certlike"].get("Certlike", {})
                 matches = {re.search(r"#\s{0,1}\d{1,4}", x) for x in fips_certlike}
                 return {"".join([x for x in match.group() if x.isdigit()]) for match in matches if match}
-            else:
-                return set()
+            return set()
 
     @dataclass(eq=True)
     class Heuristics(BaseHeuristics, ComplexSerializableType):
diff --git a/src/sec_certs/sample/sar.py b/src/sec_certs/sample/sar.py
index 8f48f417..e3d7bd2b 100644
--- a/src/sec_certs/sample/sar.py
+++ b/src/sec_certs/sample/sar.py
@@ -50,7 +50,7 @@ class SAR(ComplexSerializableType):
     @staticmethod
     def matches_re(string: str) -> bool:
         return any(
-            [re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING]
+            re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING
         )
 
     def __lt__(self, other: Any) -> bool:
diff --git a/src/sec_certs/utils/extract.py b/src/sec_certs/utils/extract.py
index 933db4b8..e6312b8d 100644
--- a/src/sec_certs/utils/extract.py
+++ b/src/sec_certs/utils/extract.py
@@ -353,7 +353,7 @@ def search_only_headers_bsi(filepath: Path):  # noqa: C901
 
             for m in re.finditer(rule_and_sep, whole_text):
                 # check if previous rules had at least one match
-                if constants.TAG_CERT_ID not in items_found.keys():
+                if constants.TAG_CERT_ID not in items_found:
                     logger.error(f"ERROR: front page not found for file: {filepath}")
 
                 match_groups = m.groups()
diff --git a/src/sec_certs/utils/helpers.py b/src/sec_certs/utils/helpers.py
index 596ecf62..c103a2d3 100644
--- a/src/sec_certs/utils/helpers.py
+++ b/src/sec_certs/utils/helpers.py
@@ -28,7 +28,11 @@ def download_file(
         time.sleep(delay)
         # See https://github.com/psf/requests/issues/3953 for header justification
         r = requests.get(
-            url, allow_redirects=True, timeout=constants.REQUEST_TIMEOUT, stream=True, headers={"Accept-Encoding": None}  # type: ignore
+            url,
+            allow_redirects=True,
+            timeout=constants.REQUEST_TIMEOUT,
+            stream=True,
+            headers={"Accept-Encoding": None},  # type: ignore
         )
         ctx: Any
         if show_progress_bar:
diff --git a/src/sec_certs/utils/pandas.py b/src/sec_certs/utils/pandas.py
index 5b0a668d..661b669c 100644
--- a/src/sec_certs/utils/pandas.py
+++ b/src/sec_certs/utils/pandas.py
@@ -141,7 +141,7 @@ def get_sar_level_from_set(sars: set[SAR], sar_family: str) -> int | None:
     Given a set of SARs and a family name, will return level of the seeked SAR from the set.
     """
     family_sars_dict = {x.family: x for x in sars} if (sars and not pd.isnull(sars)) else {}
-    if sar_family not in family_sars_dict.keys():
+    if sar_family not in family_sars_dict:
         return None
     return family_sars_dict[sar_family].level
 
@@ -211,7 +211,7 @@ def compute_cve_correlations(
     tuples = list(
         zip(n_cves_corrs, n_cves_pvalues, worst_cve_corrs, worst_cve_pvalues, avg_cve_corrs, avg_cve_pvalues, supports)
     )
-    dct = {family: correlations for family, correlations in zip(["eal"] + families, tuples)}
+    dct = dict(zip(["eal"] + families, tuples))
     df_corr = pd.DataFrame.from_dict(
         dct,
         orient="index",
diff --git a/src/sec_certs/utils/pdf.py b/src/sec_certs/utils/pdf.py
index 749a8a5a..f2c2c58e 100644
--- a/src/sec_certs/utils/pdf.py
+++ b/src/sec_certs/utils/pdf.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import glob
 import logging
 import subprocess
 from datetime import datetime, timedelta, timezone
@@ -11,6 +10,8 @@ from typing import Any
 
 import pdftotext
 import pikepdf
+import pytesseract
+from PIL import Image
 
 from sec_certs import constants
 from sec_certs.constants import (
@@ -51,13 +52,16 @@ def ocr_pdf_file(pdf_path: Path) -> str:
         )
         if ppm.returncode != 0:
             raise ValueError(f"pdftoppm failed: {ppm.returncode}")
-        for ppm_path in map(Path, glob.glob(str(tmppath / "image*.ppm"))):
+
+        for ppm_path in tmppath.rglob("image*.ppm"):
             base = ppm_path.with_suffix("")
-            tes = subprocess.run(
-                ["tesseract", "-l", "eng+deu+fra", ppm_path, base], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
-            )
-            if tes.returncode != 0:
-                raise ValueError(f"tesseract failed: {tes.returncode}")
+            content = pytesseract.image_to_string(Image.open(ppm_path), lang="eng+deu+fra")
+
+            if content:
+                with Path(base.with_suffix(".txt")).open("w") as file:
+                    file.write(content)
+            else:
+                raise ValueError(f"OCR failed for document {ppm_path}. Check document manually")
 
         contents = ""
 
diff --git a/src/sec_certs/utils/plot_utils.py b/src/sec_certs/utils/plot_utils.py
new file mode 100644
index 00000000..b9e7ac7c
--- /dev/null
+++ b/src/sec_certs/utils/plot_utils.py
@@ -0,0 +1,85 @@
+from typing import Dict, List, Tuple
+
+from networkx import DiGraph
+from pandas import DataFrame
+
+
+def get_cert_property(df: DataFrame, cert_id: int, column: str) -> str:
+    if column not in df.columns:
+        raise ValueError(f"Dataset does not have column '{column}'")
+
+    sub_df = df[df["cert_id"] == int(cert_id)]
+
+    if not sub_df.shape[0]:  # Certificate is not in the dataset
+        raise ValueError(f"Cert ID: {cert_id} not in dataset")
+
+    if sub_df.shape[0] > 1:  # There are more than one occurence with same ID
+        raise ValueError(f"Error Cert ID: {cert_id} has {sub_df.shape[0]} occurrences.")
+
+    return sub_df.iloc[0][column]
+
+
+def get_fips_cert_references_graph(
+    df: DataFrame, cert_id: int, colour_mapper: Dict[str, str]
+) -> Tuple[DiGraph, List[str]]:
+    if cert_id not in df["cert_id"].unique():
+        raise ValueError(f"Cert ID: {cert_id} is not in the dataset")
+
+    cert_id_series = df[df["cert_id"] == cert_id].iloc[0]
+    colour_map = [colour_mapper["chosen_cert_colour"]]
+    graph = DiGraph()
+    graph.add_node(cert_id)
+
+    # Display which certificates are directly referenced by the chosen certificate
+    for referenced_cert_id in cert_id_series["module_directly_referencing"]:
+        graph.add_node(referenced_cert_id)
+        graph.add_edge(cert_id, referenced_cert_id)
+        colour_map.append(colour_mapper["referencing_colour"])
+
+    # Display which certificates are directly referencing the chosen certificate
+    for referencing_cert_id in cert_id_series["module_directly_referenced_by"]:
+        graph.add_node(referencing_cert_id)
+        graph.add_edge(referencing_cert_id, cert_id)
+        colour_map.append(colour_mapper["referenced_colour"])
+
+    return graph, colour_map
+
+
+def get_most_referenced_cert_graph(df: DataFrame, status_colour_mapper: Dict[str, str]) -> Tuple[DiGraph, List[str]]:
+    graph = DiGraph()
+    colour_map = []
+    max_referenced_by_num = df["incoming_direct_references_count"].max()
+    most_referenced_certificate = df[df["incoming_direct_references_count"] == max_referenced_by_num].iloc[0]
+
+    origin_cert_id: int = most_referenced_certificate["cert_id"]
+    origin_cert_status: str = most_referenced_certificate["status"]
+    graph.add_node(origin_cert_id)
+    colour_map.append(status_colour_mapper[origin_cert_status])
+
+    for cert_id_str in most_referenced_certificate["module_directly_referenced_by"]:
+        cert_id_int = int(cert_id_str)
+        graph.add_node(cert_id_int)
+        graph.add_edge(cert_id_int, origin_cert_id)
+        cert_status: str = get_cert_property(df, cert_id_int, "status")
+        colour_map.append(status_colour_mapper[cert_status])
+
+    return graph, colour_map
+
+
+def get_most_referencing_cert_graph(df: DataFrame, status_colour_mapper: Dict[str, str]) -> Tuple[DiGraph, List[str]]:
+    graph = DiGraph()
+    colour_map = []
+    max_referencing_num = df["outgoing_direct_references_count"].max()
+    most_referencing_cert = df[df["outgoing_direct_references_count"] == max_referencing_num].iloc[0]
+    origin_cert_id = most_referencing_cert["cert_id"]
+    origin_cert_status = most_referencing_cert["status"]
+    colour_map.append(status_colour_mapper[origin_cert_status])
+
+    for cert_id_str in most_referencing_cert["module_directly_referencing"]:
+        cert_id_int = int(cert_id_str)
+        graph.add_node(cert_id_int)
+        graph.add_edge(origin_cert_id, cert_id_int)
+        cert_status: str = get_cert_property(df, cert_id_int, "status")
+        colour_map.append(status_colour_mapper[cert_status])
+
+    return graph, colour_map
diff --git a/src/sec_certs/utils/profiling.py b/src/sec_certs/utils/profiling.py
new file mode 100644
index 00000000..7da67070
--- /dev/null
+++ b/src/sec_certs/utils/profiling.py
@@ -0,0 +1,45 @@
+import gc
+from contextlib import contextmanager
+from datetime import datetime
+from functools import wraps
+from logging import Logger
+
+import psutil
+
+
+@contextmanager
+def log_stage(logger: Logger, msg: str, collect_garbage: bool = False):
+    """Contextmanager that logs a message to the logger when it is entered and exited.
+    The message has debug information about memory use. Optionally, it can
+    run garbage collection when exiting.
+    """
+    meminfo = psutil.Process().memory_full_info()
+    logger.info(f">> Starting >> {msg}")
+    logger.debug(str(meminfo))
+    start_time = datetime.now()
+
+    try:
+        yield
+    finally:
+        end_time = datetime.now()
+        duration = end_time - start_time
+        meminfo = psutil.Process().memory_full_info()
+        logger.info(f"<< Finished << {msg} ({duration})")
+        logger.debug(str(meminfo))
+
+        if collect_garbage:
+            gc.collect()
+
+
+def staged(logger: Logger, log_message: str, collect_garbage: bool = False):
+    """Like log_stage but a decorator."""
+
+    def deco(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            with log_stage(logger, log_message, collect_garbage):
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return deco