From 3738589068b5f3bdfc824d6681fae13413c8dadf Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 9 Dec 2023 13:02:19 +0100
Subject: Fix FIPS MIP introcution of "On Hold".

The computaion of flows might be more messy now.
---
 src/sec_certs/sample/fips_mip.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/sec_certs/sample/fips_mip.py b/src/sec_certs/sample/fips_mip.py
index 0720e520..71868954 100644
--- a/src/sec_certs/sample/fips_mip.py
+++ b/src/sec_certs/sample/fips_mip.py
@@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
 
 @total_ordering
 class MIPStatus(Enum):
+    ON_HOLD = "On Hold"
     IN_REVIEW = "In Review"
     REVIEW_PENDING = "Review Pending"
     COORDINATION = "Coordination"
-- 
cgit v1.3.1


From 9778786ca8ee702a349fe657b83da657c72cc2da Mon Sep 17 00:00:00 2001
From: dependabot[bot]
Date: Wed, 20 Dec 2023 21:14:20 +0000
Subject: chore(deps): bump transformers from 4.35.2 to 4.36.0 in /requirements

Bumps [transformers](https://github.com/huggingface/transformers) from 4.35.2 to 4.36.0.
- [Release notes](https://github.com/huggingface/transformers/releases)
- [Commits](https://github.com/huggingface/transformers/compare/v4.35.2...v4.36.0)

---
updated-dependencies:
- dependency-name: transformers
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>---
 requirements/all_requirements.txt | 2 +-
 requirements/nlp_requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index 22ad0f64..19643985 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -763,7 +763,7 @@ traitlets==5.13.0
     #   matplotlib-inline
     #   nbclient
     #   nbformat
-transformers==4.35.2
+transformers==4.36.0
     # via sentence-transformers
 typer==0.9.0
     # via
diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
index 14791b42..3d6671d8 100644
--- a/requirements/nlp_requirements.txt
+++ b/requirements/nlp_requirements.txt
@@ -591,7 +591,7 @@ traitlets==5.13.0
     #   jupyter-client
     #   jupyter-core
     #   matplotlib-inline
-transformers==4.35.2
+transformers==4.36.0
     # via sentence-transformers
 typer==0.9.0
     # via
-- 
cgit v1.3.1


From 7ad0eed7510a08740efd195bf4ee2f89a719dcc6 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Tue, 2 Jan 2024 17:13:08 +0100
Subject: Fix CC CSV and HTML parsing.

---
 src/sec_certs/dataset/cc.py |  7 +++++--
 src/sec_certs/sample/cc.py  | 20 +++++++++++++-------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index 04db4d7c..4eee93b6 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -178,7 +178,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     BASE_URL: ClassVar[str] = "https://www.commoncriteriaportal.org"
 
     HTML_PRODUCTS_URL = {
-        "cc_products_active.html": BASE_URL + "/products/",
+        "cc_products_active.html": BASE_URL + "/products/index.cfm",
         "cc_products_archived.html": BASE_URL + "/products/index.cfm?archived=1",
     }
     HTML_LABS_URL = {"cc_labs.html": BASE_URL + "/labs"}
@@ -362,7 +362,10 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         ]
 
         # TODO: Now skipping bad lines, smarter heuristics to be built for dumb files
-        df = pd.read_csv(file, engine="python", encoding="windows-1252", on_bad_lines="skip")
+        try:
+            df = pd.read_csv(file, engine="python", encoding="utf-8", on_bad_lines="skip")
+        except UnicodeDecodeError:
+            df = pd.read_csv(file, engine="python", encoding="windows-1252", on_bad_lines="skip")
         df = df.rename(columns=dict(zip(list(df.columns), csv_header)))
 
         df["is_maintenance"] = ~df.maintenance_title.isnull()
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 32c875c8..4cd1324d 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -486,8 +486,8 @@ class CCCertificate(
         security_level: str | set[str],
         not_valid_before: date | None,
         not_valid_after: date | None,
-        report_link: str,
-        st_link: str,
+        report_link: str | None,
+        st_link: str | None,
         cert_link: str | None,
         manufacturer_web: str | None,
         protection_profiles: set[ProtectionProfile] | None,
@@ -694,13 +694,19 @@ class CCCertificate(
         return extracted_date
 
     @staticmethod
-    def _html_row_get_report_st_links(cell: Tag) -> tuple[str, str]:
+    def _html_row_get_report_st_links(cell: Tag) -> tuple[str | None, str | None]:
         links = cell.find_all("a")
-        assert links[1].get("title").startswith("Certification Report")
-        assert links[2].get("title").startswith("Security Target")
 
-        report_link = CCCertificate.cc_url + links[1].get("href")
-        security_target_link = CCCertificate.cc_url + links[2].get("href")
+        report_link: str | None = None
+        security_target_link: str | None = None
+        for link in links:
+            title = link.get("title")
+            if not title:
+                continue
+            if title.startswith("Certification Report"):
+                report_link = CCCertificate.cc_url + link.get("href")
+            elif title.startswith("Security Target"):
+                security_target_link = CCCertificate.cc_url + link.get("href")
 
         return report_link, security_target_link
 
-- 
cgit v1.3.1


From ea830883eabb82f43275fa148ce754f9b2e3d184 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 4 Jan 2024 16:27:31 +0100
Subject: Fix CC scheme downloads.

---
 src/sec_certs/constants.py        |   9 +-
 src/sec_certs/sample/cc_scheme.py | 246 ++++++++++++++++++++++++--------------
 src/sec_certs/sample/cve.py       |   2 +-
 tests/cc/test_cc_schemes.py       |   3 +
 4 files changed, 163 insertions(+), 97 deletions(-)

diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
index 01649bb7..228a2b84 100644
--- a/src/sec_certs/constants.py
+++ b/src/sec_certs/constants.py
@@ -82,8 +82,8 @@ CC_CANADA_BASE_URL = "https://www.cyber.gc.ca"
 CC_CANADA_API_URL = CC_CANADA_BASE_URL + "/api/cccs/page/v1/get"
 CC_CANADA_CERTIFIED_URL = "/en/tools-services/common-criteria/certified-products"
 CC_CANADA_INEVAL_URL = "/en/tools-services/common-criteria/products-evaluation"
-CC_ANSSI_BASE_URL = "https://www.ssi.gouv.fr"
-CC_ANSSI_CERTIFIED_URL = CC_ANSSI_BASE_URL + "/en/products/certified-products/"
+CC_ANSSI_BASE_URL = "https://cyber.gouv.fr"
+CC_ANSSI_CERTIFIED_URL = CC_ANSSI_BASE_URL + "/produits-certifies"
 CC_BSI_BASE_URL = "https://www.bsi.bund.de/"
 CC_BSI_CERTIFIED_URL = (
     CC_BSI_BASE_URL
@@ -103,7 +103,10 @@ CC_JAPAN_ARCHIVED_SW_URL = CC_JAPAN_BASE_URL + "/software/certified-cert/archive
 CC_JAPAN_INEVAL_URL = CC_JAPAN_BASE_URL + "/prdct-in-eval/in_eval_list.html"
 CC_MALAYSIA_BASE_URL = "https://iscb.cybersecurity.my"
 CC_MALAYSIA_CERTIFIED_URL = (
-    CC_MALAYSIA_BASE_URL + "/index.php/certification/product-certification/mycc/certified-products-and-systems"
+    CC_MALAYSIA_BASE_URL + "/index.php/certification/product-certification/mycc/certified-products-and-systems-5"
+)
+CC_MALAYSIA_ARCHIVED_URL = (
+    CC_MALAYSIA_BASE_URL + "/index.php/certification/product-certification/mycc/archived-certified-products-and-systems"
 )
 CC_MALAYSIA_INEVAL_URL = (
     CC_MALAYSIA_BASE_URL
diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py
index 73946cf3..465d0dce 100644
--- a/src/sec_certs/sample/cc_scheme.py
+++ b/src/sec_certs/sample/cc_scheme.py
@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import hashlib
 import math
+import re
 import tempfile
 import warnings
 from collections.abc import Callable, Iterable
@@ -67,7 +68,7 @@ def _get(url: str, session, **kwargs) -> Response:
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=InsecureRequestWarning)
         conn = session if session else requests
-        resp = conn.get(url, headers={"User-Agent": "seccerts.org"}, verify=False, **kwargs)
+        resp = conn.get(url, headers={"User-Agent": "seccerts.org"}, verify=False, **kwargs, timeout=10)
     resp.raise_for_status()
     return resp
 
@@ -91,6 +92,7 @@ def get_australia_in_evaluation(enhanced: bool = True) -> list[dict[str, Any]]:
     :param enhanced: Whether to enhance the results by following links (slower, more data).
     :return: The entries.
     """
+    # TODO: Australian scheme is blocking our User-Agent.
     soup = _get_page(constants.CC_AUSTRALIA_INEVAL_URL)
     header = soup.find("h2", string="Products in evaluation")
     table = header.find_next_sibling("table")
@@ -207,85 +209,80 @@ def get_france_certified(enhanced: bool = True, artifacts: bool = False) -> list
     :return: The entries.
     """
     base_soup = _get_page(constants.CC_ANSSI_CERTIFIED_URL)
-    category_nav = base_soup.find("ul", class_="nav-categories")
+    pager = base_soup.find("nav", class_="pager")
+    last_page_a = re.search("[0-9]+", pager.find("a", title="Aller à la dernière page").text)
+    if not last_page_a:
+        raise ValueError
+    pages = int(last_page_a.group())
     results = []
-    for li in tqdm(category_nav.find_all("li"), desc="Get FR scheme certified."):
-        a = li.find("a")
-        url = a["href"]
-        category_name = sns(a.text)
-        soup = _get_page(urljoin(constants.CC_ANSSI_BASE_URL, url))
-        table = soup.find("table", class_="produits-liste cc")
-        if not table:
-            continue
-        tbody = table.find("tbody")
-        for tr in tqdm(tbody.find_all("tr")):
-            tds = tr.find_all("td")
-            if not tds:
-                continue
+    for page in range(pages + 1):
+        soup = _get_page(constants.CC_ANSSI_CERTIFIED_URL + f"?page={page}")
+        for row in soup.find_all("article", class_="node--type-produit-certifie-cc"):
             cert: dict[str, Any] = {
-                "product": sns(tds[0].text),
-                "vendor": sns(tds[1].text),
-                "level": sns(tds[2].text),
-                "id": sns(tds[3].text),
-                "certification_date": sns(tds[4].text),
-                "category": category_name,
-                "url": urljoin(constants.CC_ANSSI_BASE_URL, tds[0].find("a")["href"]),
+                "product": sns(row.find("h3").text),
+                "description": sns(row.find("p", class_="field-body")),
+                "url": urljoin(constants.CC_ANSSI_BASE_URL, row.find("a")["href"]),
             }
+            complement_info = row.find("div", class_="info-complement")
+            for li in complement_info.find_all("li"):
+                label = li.find("span").text
+                value = sns(li.find(text=True, recursive=False))
+                if "Commanditaire" in label:
+                    cert["sponsor"] = value
+                elif "Développeur" in label:
+                    cert["developer"] = value
+                elif "Référence du certificat" in label:
+                    cert["cert_id"] = value
+                elif "Niveau" in label:
+                    cert["level"] = value
+                elif "Date de fin de validité" in label:
+                    cert["expiration_date"] = value
             if enhanced:
                 e: dict[str, Any] = {}
                 cert_page = _get_page(cert["url"])
-                ref = cert_page.find("div", class_="ref-date")
-                for ref_li in ref.find_all("li"):
-                    title, value = (sns(span.text) for span in ref_li.find_all("span", recursive=False))
-                    if not title:
-                        continue
-                    if "Référence" in title:
-                        e["id"] = value
-                    elif "Date de certification" in title:
+                infos = cert_page.find("div", class_="product-infos-wrapper")
+                for tr in infos.find_all("tr"):
+                    label = tr.find("th").text
+                    value = sns(tr.find("td").text)
+                    if "Référence du certificat" in label:
+                        e["cert_id"] = value
+                    elif "Date de certification" in label:
                         e["certification_date"] = value
-                    elif "Date de fin de validité" in title:
+                    elif "Date de fin de validité" in label:
                         e["expiration_date"] = value
-                details = cert_page.find("div", class_="details")
-                for detail_li in details.find_all("li"):
-                    title, value = (sns(span.text) for span in detail_li.find_all("span", recursive=False))
-                    if not title:
-                        continue
-                    if "Catégorie" in title:
+                    elif "Catégorie" in label:
+                        # TODO: translate?
                         e["category"] = value
-                    elif "Référentiel" in title:
+                    elif "Référentiel" in label:
                         e["cc_version"] = value
-                    elif "Niveau" in title:
-                        e["level"] = value
-                    elif "Augmentations" in title:
-                        e["augmentations"] = value
-                    elif "Profil de protection" in title:
-                        e["protection_profile"] = value
-                    elif "Développeur" in title:
+                    elif "Développeur(s)" in label:
                         e["developer"] = value
-                    elif "Centre d'évaluation" in title:
+                    elif "Commanditaire(s)" in label:
+                        e["sponsor"] = value
+                    elif "Centre d'évaluation" in label:
                         e["evaluation_facility"] = value
-                    elif "Accords de reconnaissance" in title:
-                        e["recognition"] = value
-                e["description"] = sns(cert_page.find("div", class_="box-produit-descriptif").text)
-                links = cert_page.find("div", class_="box-produit-telechargements")
-                for link_li in links.find_all("li"):
-                    a = link_li.find("a")
-                    href = urljoin(constants.CC_ANSSI_BASE_URL, a["href"])
-                    title = sns(a.text)
-                    if not title:
-                        continue
-                    if "Rapport de certification" in title:
-                        e["report_link"] = href
+                    elif "Niveau" in label:
+                        e["level"] = value
+                    elif "Profil de protection" in label:
+                        e["protection_profile"] = value
+                    elif "Accords de reconnaissance" in label:
+                        e["mutual_recognition"] = value
+                    elif "Augmentations" in label:
+                        e["augmented"] = value
+                documents = cert_page.find("div", class_="documents")
+                for a in documents.find_all("a"):
+                    if "Rapport de certification" in a.text:
+                        e["report_link"] = urljoin(constants.CC_ANSSI_BASE_URL, a["href"])
                         if artifacts:
-                            e["report_hash"] = _get_hash(href).hex()
-                    elif "Security target" in title:
-                        e["target_link"] = href
+                            e["report_hash"] = _get_hash(e["report_link"]).hex()
+                    elif "Cible de sécurité" in a.text:
+                        e["target_link"] = urljoin(constants.CC_ANSSI_BASE_URL, a["href"])
                         if artifacts:
-                            e["target_hash"] = _get_hash(href).hex()
-                    elif "Certificat" in title:
-                        e["cert_link"] = href
+                            e["target_hash"] = _get_hash(e["target_link"]).hex()
+                    elif "Certificat" in a.text:
+                        e["cert_link"] = urljoin(constants.CC_ANSSI_BASE_URL, a["href"])
                         if artifacts:
-                            e["cert_hash"] = _get_hash(href).hex()
+                            e["cert_hash"] = _get_hash(e["cert_link"]).hex()
                 cert["enhanced"] = e
             results.append(cert)
     return results
@@ -731,40 +728,99 @@ def get_japan_in_evaluation() -> list[dict[str, Any]]:
     return results
 
 
-def get_malaysia_certified() -> list[dict[str, Any]]:
-    """
-    Get Malaysian "certified product" entries.
-
-    :return: The entries.
-    """
-    soup = _get_page(constants.CC_MALAYSIA_CERTIFIED_URL)
-    sections = soup.find("div", attrs={"itemprop": "articleBody"}).find_all("section", class_="sppb-section")
+def _get_malaysia(url, enhanced, artifacts) -> list[dict[str, Any]]:  # noqa: C901
+    soup = _get_page(url)
+    pages_re = re.search("Page [0-9]+ of ([0-9]+)", soup.find("form").text)
+    if not pages_re:
+        raise ValueError
+    total_pages = int(pages_re.group(1))
     results = []
-    for section in sections:
-        table = section.find("table")
-        if table is None:
-            continue
-        heading = section.find("h5")
-        if heading is None:
-            continue
-        category_name = sns(heading.text)
-        tbody = table.find("tbody")
-        for tr in tbody.find_all("tr", recursive=False):
-            tds = tr.find_all("td", recursive=False)
-            if len(tds) != 6:
-                continue
-            cert = {
-                "category": category_name,
-                "level": sns(tds[0].text),
-                "cert_id": sns(tds[1].text),
-                "certification_date": sns(tds[2].text),
+    for i in range(total_pages):
+        soup = _get_page(url + f"?start={i*10}")
+        table = soup.find("table", class_="directoryTable")
+        for tr in table.find_all("tr", class_="directoryRow"):
+            tds = tr.find_all("td")
+            cert: dict[str, Any] = {
+                "cert_no": sns(tds[0].text),
+                "developer": sns(tds[1].text),
+                "level": sns(tds[2].text),
                 "product": sns(tds[3].text),
-                "developer": sns(tds[4].text),
+                "certification_date": sns(tds[4].text),
+                "expiration_date": sns(tds[5].text),
+                "recognition": sns(tds[6].text),
+                "url": urljoin(constants.CC_MALAYSIA_BASE_URL, tds[7].find("a")["href"]),
             }
+            if enhanced:
+                e: dict[str, Any] = {}
+                cert_page = _get_page(cert["url"])
+                for row in cert_page.find_all("div", class_="rsform-table-row"):
+                    left = row.find("div", class_="rsform-left-col")
+                    right = row.find("div", class_="rsform-right-col")
+                    title = left.text
+                    value = sns(right.text)
+                    if "Project ID" in title:
+                        e["cert_id"] = value
+                    elif "Product Name and Version" in title:
+                        e["product"] = sns(right.text)
+                    elif "Product Sponsor / Developer" in title:
+                        e["developer"] = value
+                    elif "Category" in title:
+                        e["category"] = value
+                    elif "Product Type" in title:
+                        e["type"] = value
+                    elif "Scope" in title:
+                        e["scope"] = value
+                    elif "Product Sponsor / Developer Contact Details" in title:
+                        e["developer_contact"] = value
+                    elif "Assurance Level" in title:
+                        e["assurance_level"] = value
+                    elif "Certificate Date" in title:
+                        e["certification_date"] = value
+                    elif "Expiry Date" in title:
+                        e["expiration_date"] = value
+                    elif "Recognized By" in title:
+                        e["mutual_recognition"] = value
+                    elif "Reports" in title:
+                        for a in right.find_all("a"):
+                            if "ST" in a.text:
+                                e["target_link"] = urljoin(constants.CC_MALAYSIA_BASE_URL, a["href"])
+                                if artifacts:
+                                    e["target_hash"] = _get_hash(e["target_link"]).hex()
+                            elif "CR" in a.text:
+                                e["report_link"] = urljoin(constants.CC_MALAYSIA_BASE_URL, a["href"])
+                                if artifacts:
+                                    e["report_hash"] = _get_hash(e["report_link"]).hex()
+                    elif "Maintenance" in title:
+                        pass
+                    elif "Status" in title:
+                        e["status"] = value
+                cert["enhanced"] = e
             results.append(cert)
     return results
 
 
+def get_malaysia_certified(enhanced: bool = True, artifacts: bool = False) -> list[dict[str, Any]]:
+    """
+    Get Malaysian "certified product" entries.
+
+    :param enhanced: Whether to enhance the results by following links (slower, more data).
+    :param artifacts: Whether to download and compute artifact hashes (way slower, even more data).
+    :return: The entries.
+    """
+    return _get_malaysia(constants.CC_MALAYSIA_CERTIFIED_URL, enhanced, artifacts)
+
+
+def get_malaysia_archived(enhanced: bool = True, artifacts: bool = False) -> list[dict[str, Any]]:
+    """
+    Get Malaysian "archived product" entries.
+
+    :param enhanced: Whether to enhance the results by following links (slower, more data).
+    :param artifacts: Whether to download and compute artifact hashes (way slower, even more data).
+    :return: The entries.
+    """
+    return _get_malaysia(constants.CC_MALAYSIA_ARCHIVED_URL, enhanced, artifacts)
+
+
 def get_malaysia_in_evaluation() -> list[dict[str, Any]]:
     """
     Get Malaysian "product in evaluation" entries.
@@ -1531,7 +1587,11 @@ class CCScheme(ComplexSerializableType):
             EntryType.Certified: get_japan_certified,
             EntryType.Archived: get_japan_archived,
         },
-        "MY": {EntryType.Certified: get_malaysia_certified, EntryType.InEvaluation: get_malaysia_in_evaluation},
+        "MY": {
+            EntryType.Certified: get_malaysia_certified,
+            EntryType.Archived: get_malaysia_archived,
+            EntryType.InEvaluation: get_malaysia_in_evaluation,
+        },
         "NL": {EntryType.Certified: get_netherlands_certified, EntryType.InEvaluation: get_netherlands_in_evaluation},
         "NO": {EntryType.Certified: get_norway_certified, EntryType.Archived: get_norway_archived},
         "KO": {EntryType.Certified: get_korea_certified, EntryType.Archived: get_korea_archived},
diff --git a/src/sec_certs/sample/cve.py b/src/sec_certs/sample/cve.py
index 7f1a7cba..e1df1dbe 100644
--- a/src/sec_certs/sample/cve.py
+++ b/src/sec_certs/sample/cve.py
@@ -195,7 +195,7 @@ class CVE(PandasSerializableType, ComplexSerializableType):
 
     @staticmethod
     def parse_single_configuration(
-        configuration: dict[str, Any]
+        configuration: dict[str, Any],
     ) -> tuple[list[CPEMatchCriteria], CPEMatchCriteriaConfiguration | None]:
         if CVE.configuration_is_simple(configuration):
             return CVE.get_simple_criteria_from_cpe_matches(configuration["nodes"][0]["cpeMatch"]), None
diff --git a/tests/cc/test_cc_schemes.py b/tests/cc/test_cc_schemes.py
index 7880ca1c..984666ef 100644
--- a/tests/cc/test_cc_schemes.py
+++ b/tests/cc/test_cc_schemes.py
@@ -95,6 +95,9 @@ def test_malaysia():
     certified = CCSchemes.get_malaysia_certified()
     assert len(certified) != 0
     assert absolute_urls(certified)
+    archived = CCSchemes.get_malaysia_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
     ineval = CCSchemes.get_malaysia_in_evaluation()
     assert len(ineval) != 0
     assert absolute_urls(ineval)
-- 
cgit v1.3.1


From 784fcf6c28535cd5567f8348026cce46e23ad7c6 Mon Sep 17 00:00:00 2001
From: dependabot[bot]
Date: Thu, 11 Jan 2024 20:17:40 +0000
Subject: chore(deps): bump jinja2 from 3.1.2 to 3.1.3 in /requirements

Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3.
- [Release notes](https://github.com/pallets/jinja/releases)
- [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/jinja/compare/3.1.2...3.1.3)

---
updated-dependencies:
- dependency-name: jinja2
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>---
 requirements/all_requirements.txt  | 2 +-
 requirements/dev_requirements.txt  | 2 +-
 requirements/nlp_requirements.txt  | 2 +-
 requirements/requirements.txt      | 2 +-
 requirements/test_requirements.txt | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index 19643985..799f4f3d 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -210,7 +210,7 @@ ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
 jedi==0.19.1
     # via ipython
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   bokeh
     #   myst-parser
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index b3f20876..cf51f8c6 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -160,7 +160,7 @@ ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
 jedi==0.19.1
     # via ipython
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   myst-parser
     #   spacy
diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
index 3d6671d8..141ff340 100644
--- a/requirements/nlp_requirements.txt
+++ b/requirements/nlp_requirements.txt
@@ -161,7 +161,7 @@ ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
 jedi==0.19.1
     # via ipython
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   bokeh
     #   spacy
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 6c1e24b7..436ce736 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -78,7 +78,7 @@ ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
 jedi==0.19.1
     # via ipython
-jinja2==3.1.2
+jinja2==3.1.3
     # via spacy
 joblib==1.3.2
     # via scikit-learn
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index f20b0ebf..9c70fb80 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -86,7 +86,7 @@ ipywidgets==8.1.1
     # via sec-certs (./../pyproject.toml)
 jedi==0.19.1
     # via ipython
-jinja2==3.1.2
+jinja2==3.1.3
     # via spacy
 joblib==1.3.2
     # via scikit-learn
-- 
cgit v1.3.1


From 5bc86ca201d5ef27206da09826d9738ef4ec2338 Mon Sep 17 00:00:00 2001
From: dependabot[bot]
Date: Mon, 22 Jan 2024 22:15:06 +0000
Subject: chore(deps): bump pillow from 10.1.0 to 10.2.0 in /requirements

Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.1.0 to 10.2.0.
- [Release notes](https://github.com/python-pillow/Pillow/releases)
- [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst)
- [Commits](https://github.com/python-pillow/Pillow/compare/10.1.0...10.2.0)

---
updated-dependencies:
- dependency-name: pillow
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>---
 requirements/all_requirements.txt  | 2 +-
 requirements/dev_requirements.txt  | 2 +-
 requirements/nlp_requirements.txt  | 2 +-
 requirements/requirements.txt      | 2 +-
 requirements/test_requirements.txt | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index 799f4f3d..d607c263 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -434,7 +434,7 @@ pexpect==4.8.0
     # via ipython
 pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.1.0
+pillow==10.2.0
     # via
     #   bokeh
     #   datashader
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index cf51f8c6..b4c14f6a 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -297,7 +297,7 @@ pexpect==4.8.0
     # via ipython
 pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.1.0
+pillow==10.2.0
     # via
     #   matplotlib
     #   pikepdf
diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
index 141ff340..75caef09 100644
--- a/requirements/nlp_requirements.txt
+++ b/requirements/nlp_requirements.txt
@@ -347,7 +347,7 @@ pexpect==4.8.0
     # via ipython
 pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.1.0
+pillow==10.2.0
     # via
     #   bokeh
     #   datashader
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 436ce736..93645663 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -160,7 +160,7 @@ pexpect==4.8.0
     # via ipython
 pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.1.0
+pillow==10.2.0
     # via
     #   matplotlib
     #   pikepdf
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 9c70fb80..1922b83b 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -169,7 +169,7 @@ pexpect==4.8.0
     # via ipython
 pikepdf==8.7.1
     # via sec-certs (./../pyproject.toml)
-pillow==10.1.0
+pillow==10.2.0
     # via
     #   matplotlib
     #   pikepdf
-- 
cgit v1.3.1


From be702d4165954d1d7b3fa479251905a152af3c5c Mon Sep 17 00:00:00 2001
From: dependabot[bot]
Date: Tue, 30 Jan 2024 00:09:14 +0000
Subject: chore(deps): bump aiohttp from 3.9.0 to 3.9.2 in /requirements

Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.9.0 to 3.9.2.
- [Release notes](https://github.com/aio-libs/aiohttp/releases)
- [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst)
- [Commits](https://github.com/aio-libs/aiohttp/compare/v3.9.0...v3.9.2)

---
updated-dependencies:
- dependency-name: aiohttp
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>---
 requirements/all_requirements.txt | 2 +-
 requirements/dev_requirements.txt | 2 +-
 requirements/nlp_requirements.txt | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index d607c263..d96e2811 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -1,6 +1,6 @@
 accessible-pygments==0.0.4
     # via pydata-sphinx-theme
-aiohttp==3.9.0
+aiohttp==3.9.2
     # via
     #   datasets
     #   fsspec
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index b4c14f6a..75d23714 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -1,6 +1,6 @@
 accessible-pygments==0.0.4
     # via pydata-sphinx-theme
-aiohttp==3.9.0
+aiohttp==3.9.2
     # via
     #   datasets
     #   fsspec
diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
index 75caef09..bc558064 100644
--- a/requirements/nlp_requirements.txt
+++ b/requirements/nlp_requirements.txt
@@ -1,4 +1,4 @@
-aiohttp==3.9.0
+aiohttp==3.9.2
     # via
     #   datasets
     #   fsspec
-- 
cgit v1.3.1


From 8dd6b5b3ca7e99ce9c8750e881250d77fdeadd7f Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 1 Feb 2024 16:26:39 +0100
Subject: Add france archived list.

---
 src/sec_certs/constants.py        |   1 +
 src/sec_certs/sample/cc_scheme.py | 128 +++++++++++++++++++++++++++++---------
 tests/cc/test_cc_schemes.py       |   3 +
 3 files changed, 102 insertions(+), 30 deletions(-)

diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
index 228a2b84..620f984f 100644
--- a/src/sec_certs/constants.py
+++ b/src/sec_certs/constants.py
@@ -84,6 +84,7 @@ CC_CANADA_CERTIFIED_URL = "/en/tools-services/common-criteria/certified-products
 CC_CANADA_INEVAL_URL = "/en/tools-services/common-criteria/products-evaluation"
 CC_ANSSI_BASE_URL = "https://cyber.gouv.fr"
 CC_ANSSI_CERTIFIED_URL = CC_ANSSI_BASE_URL + "/produits-certifies"
+CC_ANSSI_ARCHIVED_URL = CC_ANSSI_BASE_URL + "/produits-certifies-archives"
 CC_BSI_BASE_URL = "https://www.bsi.bund.de/"
 CC_BSI_CERTIFIED_URL = (
     CC_BSI_BASE_URL
diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py
index 465d0dce..ee09c726 100644
--- a/src/sec_certs/sample/cc_scheme.py
+++ b/src/sec_certs/sample/cc_scheme.py
@@ -31,6 +31,7 @@ __all__ = [
     "get_canada_certified",
     "get_canada_in_evaluation",
     "get_france_certified",
+    "get_france_archived",
     "get_germany_certified",
     "get_india_certified",
     "get_india_archived",
@@ -68,7 +69,13 @@ def _get(url: str, session, **kwargs) -> Response:
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=InsecureRequestWarning)
         conn = session if session else requests
-        resp = conn.get(url, headers={"User-Agent": "seccerts.org"}, verify=False, **kwargs, timeout=10)
+        resp = conn.get(
+            url,
+            headers={"User-Agent": "seccerts.org"},
+            verify=False,
+            **kwargs,
+            timeout=10,
+        )
     resp.raise_for_status()
     return resp
 
@@ -85,7 +92,9 @@ def _get_hash(url: str, session=None) -> bytes:
     return h.digest()
 
 
-def get_australia_in_evaluation(enhanced: bool = True) -> list[dict[str, Any]]:  # noqa: C901
+def get_australia_in_evaluation(  # noqa: C901
+    enhanced: bool = True,
+) -> list[dict[str, Any]]:
     """
     Get Australia "products in evaluation" entries.
 
@@ -156,7 +165,10 @@ def get_canada_certified() -> list[dict[str, Any]]:
 
     :return: The entries.
     """
-    resp = _get(constants.CC_CANADA_API_URL + f"?lang=en&url={constants.CC_CANADA_CERTIFIED_URL}", None)
+    resp = _get(
+        constants.CC_CANADA_API_URL + f"?lang=en&url={constants.CC_CANADA_CERTIFIED_URL}",
+        None,
+    )
     html_data = resp.json()["response"]["page"]["body"][0]
     soup = BeautifulSoup(html_data, "html5lib")
     tbody = soup.find("table").find("tbody")
@@ -181,7 +193,10 @@ def get_canada_in_evaluation() -> list[dict[str, Any]]:
 
     :return: The entries.
     """
-    resp = _get(constants.CC_CANADA_API_URL + f"?lang=en&url={constants.CC_CANADA_INEVAL_URL}", None)
+    resp = _get(
+        constants.CC_CANADA_API_URL + f"?lang=en&url={constants.CC_CANADA_INEVAL_URL}",
+        None,
+    )
     html_data = resp.json()["response"]["page"]["body"][0]
     soup = BeautifulSoup(html_data, "html5lib")
     tbody = soup.find("table").find("tbody")
@@ -200,15 +215,8 @@ def get_canada_in_evaluation() -> list[dict[str, Any]]:
     return results
 
 
-def get_france_certified(enhanced: bool = True, artifacts: bool = False) -> list[dict[str, Any]]:  # noqa: C901
-    """
-    Get French "certified product" entries.
-
-    :param enhanced: Whether to enhance the results by following links (slower, more data).
-    :param artifacts: Whether to download and compute artifact hashes (way slower, even more data).
-    :return: The entries.
-    """
-    base_soup = _get_page(constants.CC_ANSSI_CERTIFIED_URL)
+def _get_france(url, enhanced, artifacts) -> list[dict[str, Any]]:  # noqa: C901
+    base_soup = _get_page(url)
     pager = base_soup.find("nav", class_="pager")
     last_page_a = re.search("[0-9]+", pager.find("a", title="Aller à la dernière page").text)
     if not last_page_a:
@@ -216,13 +224,15 @@ def get_france_certified(enhanced: bool = True, artifacts: bool = False) -> list
     pages = int(last_page_a.group())
     results = []
     for page in range(pages + 1):
-        soup = _get_page(constants.CC_ANSSI_CERTIFIED_URL + f"?page={page}")
+        soup = _get_page(url + f"?page={page}")
         for row in soup.find_all("article", class_="node--type-produit-certifie-cc"):
             cert: dict[str, Any] = {
                 "product": sns(row.find("h3").text),
-                "description": sns(row.find("p", class_="field-body")),
                 "url": urljoin(constants.CC_ANSSI_BASE_URL, row.find("a")["href"]),
             }
+            description_para = row.find("p", class_="field-body")
+            if description_para:
+                cert["description"] = sns(description_para.text)
             complement_info = row.find("div", class_="info-complement")
             for li in complement_info.find_all("li"):
                 label = li.find("span").text
@@ -288,7 +298,31 @@ def get_france_certified(enhanced: bool = True, artifacts: bool = False) -> list
     return results
 
 
-def get_germany_certified(enhanced: bool = True, artifacts: bool = False) -> list[dict[str, Any]]:  # noqa: C901
+def get_france_certified(enhanced: bool = True, artifacts: bool = False) -> list[dict[str, Any]]:  # noqa: C901
+    """
+    Get French "certified product" entries.
+
+    :param enhanced: Whether to enhance the results by following links (slower, more data).
+    :param artifacts: Whether to download and compute artifact hashes (way slower, even more data).
+    :return: The entries.
+    """
+    return _get_france(constants.CC_ANSSI_CERTIFIED_URL, enhanced, artifacts)
+
+
+def get_france_archived(enhanced: bool = True, artifacts: bool = False) -> list[dict[str, Any]]:  # noqa: C901
+    """
+    Get French "archived product" entries.
+
+    :param enhanced: Whether to enhance the results by following links (slower, more data).
+    :param artifacts: Whether to download and compute artifact hashes (way slower, even more data).
+    :return: The entries.
+    """
+    return _get_france(constants.CC_ANSSI_ARCHIVED_URL, enhanced, artifacts)
+
+
+def get_germany_certified(  # noqa: C901
+    enhanced: bool = True, artifacts: bool = False
+) -> list[dict[str, Any]]:
     """
     Get German "certified product" entries.
 
@@ -736,7 +770,7 @@ def _get_malaysia(url, enhanced, artifacts) -> list[dict[str, Any]]:  # noqa: C9
     total_pages = int(pages_re.group(1))
     results = []
     for i in range(total_pages):
-        soup = _get_page(url + f"?start={i*10}")
+        soup = _get_page(url + f"?start={i * 10}")
         table = soup.find("table", class_="directoryTable")
         for tr in table.find_all("tr", class_="directoryRow"):
             tds = tr.find_all("td")
@@ -846,7 +880,9 @@ def get_malaysia_in_evaluation() -> list[dict[str, Any]]:
     return results
 
 
-def get_netherlands_certified(artifacts: bool = False) -> list[dict[str, Any]]:  # noqa: C901
+def get_netherlands_certified(  # noqa: C901
+    artifacts: bool = False,
+) -> list[dict[str, Any]]:
     """
     Get Dutch "certified product" entries.
 
@@ -916,7 +952,9 @@ def get_netherlands_in_evaluation() -> list[dict[str, Any]]:
     return results
 
 
-def _get_norway(url: str, enhanced: bool, artifacts: bool) -> list[dict[str, Any]]:  # noqa: C901
+def _get_norway(  # noqa: C901
+    url: str, enhanced: bool, artifacts: bool
+) -> list[dict[str, Any]]:
     soup = _get_page(url)
     results = []
     for tr in soup.find_all("tr", class_="certified-product"):
@@ -1013,7 +1051,9 @@ def get_norway_archived(enhanced: bool = True, artifacts: bool = False) -> list[
     return _get_norway(constants.CC_NORWAY_ARCHIVED_URL, enhanced, artifacts)
 
 
-def _get_korea(product_class: int, enhanced: bool, artifacts: bool) -> list[dict[str, Any]]:  # noqa: C901
+def _get_korea(  # noqa: C901
+    product_class: int, enhanced: bool, artifacts: bool
+) -> list[dict[str, Any]]:
     session = requests.session()
     session.get(constants.CC_KOREA_EN_URL)
     # Get base page
@@ -1177,7 +1217,10 @@ def _get_singapore(url: str, artifacts: bool) -> list[dict[str, Any]]:
                 "cert_title": obj["certificate"]["title"],
                 "cert_link": urljoin(constants.CC_SINGAPORE_BASE_URL, obj["certificate"]["mediaUrl"]),
                 "report_title": obj["certificationReport"]["title"],
-                "report_link": urljoin(constants.CC_SINGAPORE_BASE_URL, obj["certificationReport"]["mediaUrl"]),
+                "report_link": urljoin(
+                    constants.CC_SINGAPORE_BASE_URL,
+                    obj["certificationReport"]["mediaUrl"],
+                ),
                 "target_title": obj["securityTarget"]["title"],
                 "target_link": urljoin(constants.CC_SINGAPORE_BASE_URL, obj["securityTarget"]["mediaUrl"]),
             }
@@ -1269,7 +1312,9 @@ def get_spain_certified() -> list[dict[str, Any]]:
     return results
 
 
-def _get_sweden(url: str, enhanced: bool, artifacts: bool) -> list[dict[str, Any]]:  # noqa: C901
+def _get_sweden(  # noqa: C901
+    url: str, enhanced: bool, artifacts: bool
+) -> list[dict[str, Any]]:
     soup = _get_page(url)
     nav = soup.find("main").find("nav", class_="component-nav-box__list")
     results = []
@@ -1400,7 +1445,9 @@ def get_turkey_certified() -> list[dict[str, Any]]:
     return results
 
 
-def get_usa_certified(enhanced: bool = True, artifacts: bool = False) -> list[dict[str, Any]]:  # noqa: C901
+def get_usa_certified(  # noqa: C901
+    enhanced: bool = True, artifacts: bool = False
+) -> list[dict[str, Any]]:
     """
     Get American "certified product" entries.
 
@@ -1577,11 +1624,23 @@ class CCScheme(ComplexSerializableType):
 
     methods: ClassVar[dict[str, dict[EntryType, Callable]]] = {
         "AU": {EntryType.InEvaluation: get_australia_in_evaluation},
-        "CA": {EntryType.InEvaluation: get_canada_in_evaluation, EntryType.Certified: get_canada_certified},
-        "FR": {EntryType.Certified: get_france_certified},
+        "CA": {
+            EntryType.InEvaluation: get_canada_in_evaluation,
+            EntryType.Certified: get_canada_certified,
+        },
+        "FR": {
+            EntryType.Certified: get_france_certified,
+            EntryType.Archived: get_france_archived,
+        },
         "DE": {EntryType.Certified: get_germany_certified},
-        "IN": {EntryType.Certified: get_india_certified, EntryType.Archived: get_india_archived},
-        "IT": {EntryType.Certified: get_italy_certified, EntryType.InEvaluation: get_italy_in_evaluation},
+        "IN": {
+            EntryType.Certified: get_india_certified,
+            EntryType.Archived: get_india_archived,
+        },
+        "IT": {
+            EntryType.Certified: get_italy_certified,
+            EntryType.InEvaluation: get_italy_in_evaluation,
+        },
         "JP": {
             EntryType.InEvaluation: get_japan_in_evaluation,
             EntryType.Certified: get_japan_certified,
@@ -1592,9 +1651,18 @@ class CCScheme(ComplexSerializableType):
             EntryType.Archived: get_malaysia_archived,
             EntryType.InEvaluation: get_malaysia_in_evaluation,
         },
-        "NL": {EntryType.Certified: get_netherlands_certified, EntryType.InEvaluation: get_netherlands_in_evaluation},
-        "NO": {EntryType.Certified: get_norway_certified, EntryType.Archived: get_norway_archived},
-        "KO": {EntryType.Certified: get_korea_certified, EntryType.Archived: get_korea_archived},
+        "NL": {
+            EntryType.Certified: get_netherlands_certified,
+            EntryType.InEvaluation: get_netherlands_in_evaluation,
+        },
+        "NO": {
+            EntryType.Certified: get_norway_certified,
+            EntryType.Archived: get_norway_archived,
+        },
+        "KO": {
+            EntryType.Certified: get_korea_certified,
+            EntryType.Archived: get_korea_archived,
+        },
         "SG": {
             EntryType.InEvaluation: get_singapore_in_evaluation,
             EntryType.Certified: get_singapore_certified,
diff --git a/tests/cc/test_cc_schemes.py b/tests/cc/test_cc_schemes.py
index 984666ef..a9e2105c 100644
--- a/tests/cc/test_cc_schemes.py
+++ b/tests/cc/test_cc_schemes.py
@@ -47,6 +47,9 @@ def test_anssi():
     certified = CCSchemes.get_france_certified()
     assert len(certified) != 0
     assert absolute_urls(certified)
+    archived = CCSchemes.get_france_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
 
 
 @pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
-- 
cgit v1.3.1


From 3a08258b42734f25c1cb9f5b66ad9c8b12247d0a Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 1 Feb 2024 17:17:13 +0100
Subject: Fix Turkish scheme test xfail.

---
 src/sec_certs/sample/cc_scheme.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py
index ee09c726..41b16aec 100644
--- a/src/sec_certs/sample/cc_scheme.py
+++ b/src/sec_certs/sample/cc_scheme.py
@@ -1420,8 +1420,7 @@ def get_turkey_certified() -> list[dict[str, Any]]:
     with tempfile.TemporaryDirectory() as tmpdir:
         pdf_path = Path(tmpdir) / "turkey.pdf"
         resp = requests.get(constants.CC_TURKEY_ARCHIVED_URL)
-        if resp.status_code != requests.codes.ok:
-            raise ValueError(f"Unable to download: status={resp.status_code}")
+        resp.raise_for_status()
         with pdf_path.open("wb") as f:
             f.write(resp.content)
         dfs = tabula.read_pdf(str(pdf_path), pages="all")
-- 
cgit v1.3.1


From 5a4fea7a41bbedb9bcdbddbf5ec8dc12b8021b53 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 1 Feb 2024 17:04:12 +0100
Subject: Use ConfigDict in pydantic.

---
 src/sec_certs/configuration.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/sec_certs/configuration.py b/src/sec_certs/configuration.py
index 3ebb22bd..8f26bafd 100644
--- a/src/sec_certs/configuration.py
+++ b/src/sec_certs/configuration.py
@@ -6,7 +6,7 @@ from typing import Literal, Optional
 
 import yaml
 from pydantic import AnyHttpUrl, Field
-from pydantic_settings import BaseSettings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class Configuration(BaseSettings):
@@ -15,8 +15,7 @@ class Configuration(BaseSettings):
     While not a singleton, the `config` instance from this module is meant to be primarily used.
     """
 
-    class Config:
-        env_prefix = "seccerts_"
+    model_config = SettingsConfigDict(env_prefix="seccerts_")
 
     log_filepath: Path = Field(
         "./cert_processing_log.log",
-- 
cgit v1.3.1


From e5de942140115f70aa08c68f594b2619779c16cc Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 1 Feb 2024 17:05:28 +0100
Subject: Move to model_validate in pydantic.

---
 src/sec_certs/configuration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sec_certs/configuration.py b/src/sec_certs/configuration.py
index 8f26bafd..9895fa53 100644
--- a/src/sec_certs/configuration.py
+++ b/src/sec_certs/configuration.py
@@ -138,7 +138,7 @@ class Configuration(BaseSettings):
         """
         with Path(yaml_path).open("r") as handle:
             data = yaml.safe_load(handle)
-        other_cfg = Configuration.parse_obj(data)
+        other_cfg = Configuration.model_validate(data)
         keys_to_rewrite = set(data.keys()).union(other_cfg._get_nondefault_keys())
         self._set_attrs_from_cfg(other_cfg, keys_to_rewrite)
 
-- 
cgit v1.3.1


From 5e12aff90834e0896f15a2f86559c0abdb4cecd7 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 1 Feb 2024 17:07:13 +0100
Subject: Move __fields__ use to model_fields.

---
 src/sec_certs/configuration.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/sec_certs/configuration.py b/src/sec_certs/configuration.py
index 9895fa53..81fac197 100644
--- a/src/sec_certs/configuration.py
+++ b/src/sec_certs/configuration.py
@@ -121,12 +121,12 @@ class Configuration(BaseSettings):
         """
         Returns keys of the config that have non-default value, i.e. were provided as kwargs, env. vars. or additionaly set.
         """
-        return {key for key, value in Configuration.__fields__.items() if getattr(self, key) != value.default}
+        return {key for key, value in Configuration.model_fields.items() if getattr(self, key) != value.default}
 
     def _set_attrs_from_cfg(self, other_cfg: Configuration, fields_to_set: set[str] | None) -> None:
         if not fields_to_set:
-            fields_to_set = set(Configuration.__fields__.keys())
-        for field in [x for x in other_cfg.__fields__ if x in fields_to_set]:
+            fields_to_set = set(Configuration.model_fields.keys())
+        for field in [x for x in other_cfg.model_fields if x in fields_to_set]:
             setattr(self, field, getattr(other_cfg, field))
 
     def load_from_yaml(self, yaml_path: str | Path) -> None:
-- 
cgit v1.3.1


From 81a6036cfe3a232e0153073b50f24ffbbf7de2af Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 1 Feb 2024 17:08:27 +0100
Subject: Fix pandas deprecation in to_datetime.

---
 src/sec_certs/dataset/cc.py   | 6 +++---
 src/sec_certs/dataset/fips.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index 4eee93b6..398f2435 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -83,8 +83,8 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=CCCertificate.pandas_columns)
         df = df.set_index("dgst")
 
-        df.not_valid_before = pd.to_datetime(df.not_valid_before, infer_datetime_format=True, errors="coerce")
-        df.not_valid_after = pd.to_datetime(df.not_valid_after, infer_datetime_format=True, errors="coerce")
+        df.not_valid_before = pd.to_datetime(df.not_valid_before, errors="coerce")
+        df.not_valid_after = pd.to_datetime(df.not_valid_after, errors="coerce")
         df = df.astype(
             {"category": "category", "status": "category", "scheme": "category", "cert_lab": "category"}
         ).fillna(value=np.nan)
@@ -905,7 +905,7 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType):
         df = df.set_index("dgst")
         df.index.name = "dgst"
 
-        df.maintenance_date = pd.to_datetime(df.maintenance_date, infer_datetime_format=True, errors="coerce")
+        df.maintenance_date = pd.to_datetime(df.maintenance_date, errors="coerce")
         return df.fillna(value=np.nan)
 
     @classmethod
diff --git a/src/sec_certs/dataset/fips.py b/src/sec_certs/dataset/fips.py
index a3d24b15..77f38754 100644
--- a/src/sec_certs/dataset/fips.py
+++ b/src/sec_certs/dataset/fips.py
@@ -326,8 +326,8 @@ class FIPSDataset(Dataset[FIPSCertificate, FIPSAuxiliaryDatasets], ComplexSerial
         df = pd.DataFrame([x.pandas_tuple for x in self.certs.values()], columns=FIPSCertificate.pandas_columns)
         df = df.set_index("dgst")
 
-        df.date_validation = pd.to_datetime(df.date_validation, infer_datetime_format=True, errors="coerce")
-        df.date_sunset = pd.to_datetime(df.date_sunset, infer_datetime_format=True, errors="coerce")
+        df.date_validation = pd.to_datetime(df.date_validation, errors="coerce")
+        df.date_sunset = pd.to_datetime(df.date_sunset, errors="coerce")
 
         # Manually delete one certificate with bad embodiment (seems to have many blank fields)
         df = df.loc[~(df.embodiment == "*")]
-- 
cgit v1.3.1


From 748d259c146cabbbe64808cc3ae4620ab1374855 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 1 Feb 2024 17:16:02 +0100
Subject: Fix bs4 deprecation.

---
 src/sec_certs/sample/cc_scheme.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sec_certs/sample/cc_scheme.py b/src/sec_certs/sample/cc_scheme.py
index 41b16aec..8d18b5d0 100644
--- a/src/sec_certs/sample/cc_scheme.py
+++ b/src/sec_certs/sample/cc_scheme.py
@@ -236,7 +236,7 @@ def _get_france(url, enhanced, artifacts) -> list[dict[str, Any]]:  # noqa: C901
             complement_info = row.find("div", class_="info-complement")
             for li in complement_info.find_all("li"):
                 label = li.find("span").text
-                value = sns(li.find(text=True, recursive=False))
+                value = sns(li.find(string=True, recursive=False))
                 if "Commanditaire" in label:
                     cert["sponsor"] = value
                 elif "Développeur" in label:
-- 
cgit v1.3.1


From 328ca13e97fb2bc3c49ddcd0e5ec1b45300b8127 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Fri, 2 Feb 2024 16:31:47 +0100
Subject: Fix cert_lab parsing.

---
 src/sec_certs/rules.yaml   | 42 +++++++++++++++++++++---------------------
 src/sec_certs/sample/cc.py |  2 +-
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 8973b015..3cce1ab8 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -8,8 +8,8 @@ cc_cert_id:
     - "BSI-DSZ-CC-[0-9]+?-[0-9]+"
     - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+-[0-9]+"
     - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+"
-    - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(-[0-9][0-9][0-9][0-9])*"  # German BSI (number + version + year or without year)
-    - "BSI-DSZ-CC-[0-9]+-[0-9][0-9][0-9][0-9]"  # German BSI (number + year, no version)
+    - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(-[0-9]{4})*"  # German BSI (number + version + year or without year)
+    - "BSI-DSZ-CC-[0-9]+-[0-9]{4}"  # German BSI (number + year, no version)
     - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(?!-)"  # German BSI (number + version but no year => no - after version)
     # - "BSI-DSZ-CC-[0-9]+"  # Maybe?
   FR:
@@ -21,8 +21,8 @@ cc_cert_id:
     - "Certification Report [0-9]+/[0-9]+" # French or Australia! Solved because we limit ourselves to scheme when doing heuristics.
     - "Rapport de certification [0-9]+/[0-9]+"  # French
   NL:
-    - "NSCIB-CC-[0-9][0-9][0-9][0-9].+?"  # Netherlands
-    - "NSCIB-CC-[0-9][0-9][0-9][0-9][0-9]*-CR"  # Netherlands
+    - "NSCIB-CC-[0-9]{4}.+?"  # Netherlands
+    - "NSCIB-CC-[0-9]{4}[0-9]*-CR"  # Netherlands
     - "NSCIB-CC-[0-9][0-9]-[0-9]+?-CR[0-9]+?"  # Netherlands
     - "NSCIB-CC-[0-9][0-9]-[0-9]+(-CR[0-9]+)*"  # Netherlands (old number NSCIB-CC-05-6609 or NSCIB-CC-05-6609-CR)
     - "NSCIB-CC-[0-9]+-CR[0-9]*"  # Netherlands (new number NSCIB-CC-111441-CR NSCIB-CC-111441-CR1)
@@ -1083,39 +1083,39 @@ fips_security_level:
 #####
 fips_certlike:
   Certlike:
-      # --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) ---
-      # + added (and #id) everywhere
+    # --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) ---
+    # + added (and #id) everywhere
     - "HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\\(?(?: |hardware|firmware)*?[\\s(\\[]*?(?:#|cert\\.?|Cert\\.?|Certificate|sample)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:[\\s#]*and[\\s#]*\\d+)?"
-      # --- same as above, without hw or fw ---
+    # --- same as above, without hw or fw ---
     - "HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})"
-      # --- SHS/A - (bits) (method) ((cert #) numbers) ---
+    # --- SHS/A - (bits) (method) ((cert #) numbers) ---
     - "SH[SA][-– 123]*(?:;|\\/|160|224|256|384|512)?(?:[\\s(\\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\\s,]*?[\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:\\)?\\[#?\\d+\\])?(?:[\\s#]*?and[\\s#]*?\\d+)?"
-      # --- RSA (bits) (method) ((cert #)) ---
+    # --- RSA (bits) (method) ((cert #)) ---
     - "RSA(?:[-– ]*(?:;|\\/|512|768|1024|1280|1536|2048|3072|4096|8192)\\s\\(\\[]*?(?:(?:;|\\/|KAT|Verify|PSS|\\s)*?)?[\\s,]*?[\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})"
-      # --- RSA (SSA) (PKCS) (version) (#) ---
+    # --- RSA (SSA) (PKCS) (version) (#) ---
     - "(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\\s?#?\\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\\s#]*?(\\d{1,4})?"
-      # --- AES (bits) (method) ((cert #)) ---
+    # --- AES (bits) (method) ((cert #)) ---
     - "AES[-– ]*((?: |;|\\/|bit|key|128|192|256|CBC)*(?: |\\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:\\)?[\\s#]*?\\[#?\\d+\\])?(?:[\\s#]*?and[\\s#]*?(\\d+))?"
-      # --- Diffie Helman (CVL) ((cert #)) ---
+    # --- Diffie Helman (CVL) ((cert #)) ---
     - "Diffie[-– ]*Hellman[,\\s(\\[]*?(?:CVL|\\s)*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?[\\s#]*?(\\d{1,4})"
-      # --- DRBG (bits) (method) (cert #) ---
+    # --- DRBG (bits) (method) (cert #) ---
     - "DRBG[ –-]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})"
-      # --- DES (bits) (method) (cert #)
+    # --- DES (bits) (method) (cert #)
     - "DES[ –-]*((?:;|\\/|160|224|256|384|512)?(?:;|\\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\\d(?: and \\d)? keying options?))*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)*?[\\s#]*?)?[\\s#]*?(\\d{1,4})(?:[\\s#]*?and[\\s#]*?(\\d+))?"
-      # --- DSA (bits) (method) (cert #)
+    # --- DSA (bits) (method) (cert #)
     - "DSA[ –-]*((?:;|\\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\\s(\\[]*?(?:#|cert\\.?|sample|Cert\\.?|Certificate)?[\\s#]*?)?[\\s#]*?(\\d{1,4})"
-      # --- platforms (#)+ - this is used in modification history ---
+    # --- platforms (#)+ - this is used in modification history ---
     - "[Pp]latforms? #\\d+(?:#\\d+|,| |-|and)*[^\\n]*"
-      # --- CVL (#) ---
+    # --- CVL (#) ---
     - "CVL[\\s#]*?(\\d{1,4})"
-      # --- PAA (#) ---
+    # --- PAA (#) ---
     - "PAA[: #]*?\\d{1,4}"
-      # --- (#) Type ---
+    # --- (#) Type ---
     - "(?:#|cert\\.?|sample|Cert\\.?|Certificate)[\\s#]*?(\\d+)?\\s*?(?:AES|SHS|SHA|RSA|HMAC|Diffie-Hellman|DRBG|DES|CVL)"
-      # --- PKCS (#) ---
+    # --- PKCS (#) ---
     - "PKCS[\\s]?#?\\d+"
     - "PKSC[\\s]?#?\\d+"  # typo, #625
-      # --- # C and # A (just in case) ---
+    # --- # C and # A (just in case) ---
     - "#\\s+?[Cc]\\d+"
     - "#\\s+?[Aa]\\d+"
 
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 4cd1324d..6ee2bd1b 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -292,7 +292,7 @@ class CCCertificate(
             labs = [
                 data["cert_lab"].split(" ")[0].upper()
                 for data in [self.bsi_data, self.anssi_data, self.niap_data, self.nscib_data, self.canada_data]
-                if data
+                if data and "cert_lab" in data
             ]
             return labs if labs else None
 
-- 
cgit v1.3.1


From 69e21289a45f3d5f8cc9150aac84b496e89b97b4 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Fri, 2 Feb 2024 18:33:51 +0100
Subject: Improve NL cert_id rules.

---
 src/sec_certs/rules.yaml                  | 14 ++++++--------
 src/sec_certs/sample/cc_certificate_id.py |  2 +-
 tests/cc/test_cc_misc.py                  |  6 ++++++
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 3cce1ab8..b0d421f1 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -21,14 +21,12 @@ cc_cert_id:
     - "Certification Report [0-9]+/[0-9]+" # French or Australia! Solved because we limit ourselves to scheme when doing heuristics.
     - "Rapport de certification [0-9]+/[0-9]+"  # French
   NL:
-    - "NSCIB-CC-[0-9]{4}.+?"  # Netherlands
-    - "NSCIB-CC-[0-9]{4}[0-9]*-CR"  # Netherlands
-    - "NSCIB-CC-[0-9][0-9]-[0-9]+?-CR[0-9]+?"  # Netherlands
-    - "NSCIB-CC-[0-9][0-9]-[0-9]+(-CR[0-9]+)*"  # Netherlands (old number NSCIB-CC-05-6609 or NSCIB-CC-05-6609-CR)
-    - "NSCIB-CC-[0-9]+-CR[0-9]*"  # Netherlands (new number NSCIB-CC-111441-CR NSCIB-CC-111441-CR1)
-    - "NSCIB-CC-[0-9]+-MA[0-9]*"  # Netherlands (new number NSCIB-CC-222073-MA NSCIB-CC-200716-MA2)
-    - "NSCIB-CC-[0-9][0-9]-[0-9]+"  # Netherlands   (old number NSCIB-CC-05-6609)
-    - "NSCIB-CC-[0-9][0-9]-[0-9]+-CR[0-9]+"  # Netherlands (NSCIB-CC-year2digits-number-CR)
+    - "(?:NSCIB-|CC-|NSCIB-CC-)(?P<core>((?P<year>[0-9]{2})-)?(?:-?[0-9]+)+)(?:-?(?P<doc>(?:CR|MA|MR)[0-9]*))?"
+    # Examples:
+    # NSCIB-CC-22-0428888-CR2  (with year=22 and CR2)
+    # NSCIB-CC-228723-CR  (no year)
+    # CC-16-31801-CR4  (no NSCIB)
+    # NSCIB-CC-98209   (no year, no CR)
   "NO":
     - "SERTIT-[0-9]+"  # Norway
   US:
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 254c8e05..74cba680 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -126,7 +126,7 @@ class CertificateId:
         new_cert_id = self.clean
         if new_cert_id.startswith("CC-"):
             new_cert_id = f"NSCIB-{new_cert_id}"
-        if not new_cert_id.endswith("-CR"):
+        if not re.match(".*-(CR|MA|MR)[0-9]*$", new_cert_id):
             new_cert_id = f"{new_cert_id}-CR"
         return new_cert_id
 
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index b0932482..ae9a83aa 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -45,3 +45,9 @@ def test_canonicalize_jp():
 
 def test_canonicalize_no():
     assert canonicalize("SERTIT-12", "NO") == "SERTIT-012"
+
+
+def test_canonicalize_nl():
+    assert canonicalize("NSCIB-CC-22-0428888-CR2", "NL") == "NSCIB-CC-22-0428888-CR2"
+    assert canonicalize("NSCIB-CC-22-0428888", "NL") == "NSCIB-CC-22-0428888-CR"
+    assert canonicalize("CC-22-0428888", "NL") == "NSCIB-CC-22-0428888-CR"
-- 
cgit v1.3.1


From c380870abd04c84da69a9dd438abf22d430568a8 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Fri, 2 Feb 2024 19:11:22 +0100
Subject: Cleanup ANSSI rules.

---
 src/sec_certs/rules.yaml                  | 18 ++++++++++--------
 src/sec_certs/sample/cc_certificate_id.py | 28 ++++++++++++++--------------
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index b0d421f1..87acd04b 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -13,13 +13,15 @@ cc_cert_id:
     - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(?!-)"  # German BSI (number + version but no year => no - after version)
     # - "BSI-DSZ-CC-[0-9]+"  # Maybe?
   FR:
-    - "ANSS[Ii](?:-|-CC-|-CC )[0-9]{4}/[0-9]+(v[1-9])?"  # French
-    - "ANSS[Ii]-CC[ -][0-9]{4}[/-_][0-9][0-9]+(?!-M|-S|-R)"  # French (/two or more digits then NOT -M or -S)
-    - "ANSS[Ii]-CC[ -][0-9]{4}[/-_][0-9]+(?:v[0-9])?[_/-][MSR][0-9]+"  # French, maintenance or surveillance report (ANSSI-CC-2014_46_M01)
-    # 'ANSSI-CC-CER-F-.+?', # French
-    - "DCSS[Ii]-[0-9]+/[0-9]+"  # French (DCSSI-2009/07)
-    - "Certification Report [0-9]+/[0-9]+" # French or Australia! Solved because we limit ourselves to scheme when doing heuristics.
-    - "Rapport de certification [0-9]+/[0-9]+"  # French
+    - "DCSS[Ii]-(?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)([vV](?P<version>[0-9]))?"
+    - "Certification Report (?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)([vV](?P<version>[0-9]))?"
+    - "Rapport de certification (?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)([vV](?P<version>[0-9]))?"
+    - "ANSS[Ii](?:-CC)?[ -](?P<year>[0-9]{2,4})[/_-](?P<counter>[0-9]+)(?:-(?P<doc>(?:[MSR][0-9]+)))?([vV](?P<version>[0-9]))?"
+    # Examples:
+    # DCSSI-2009/07
+    # ANSSI-CC 2001/02-R01
+    # Rapport de certification 2001/02v2
+    # Certification Report 2003/20
   NL:
     - "(?:NSCIB-|CC-|NSCIB-CC-)(?P<core>((?P<year>[0-9]{2})-)?(?:-?[0-9]+)+)(?:-?(?P<doc>(?:CR|MA|MR)[0-9]*))?"
     # Examples:
@@ -28,7 +30,7 @@ cc_cert_id:
     # CC-16-31801-CR4  (no NSCIB)
     # NSCIB-CC-98209   (no year, no CR)
   "NO":
-    - "SERTIT-[0-9]+"  # Norway
+    - "SERTIT-(?P<counter>[0-9]+)"  # Norway
   US:
     - "CCEVS-VR-(?:CC-|VID)?[0-9]+-[0-9]+[a-z]?(?:-[0-9]+)?"  # US NSA (CCEVS-VR-10884-2018 CCEVS-VR-VID10877-2018)
   CA:
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 74cba680..9bce0764 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -3,6 +3,8 @@ from __future__ import annotations
 import re
 from dataclasses import dataclass
 
+from sec_certs.cert_rules import rules
+
 
 @dataclass(eq=True, frozen=True)
 class CertificateId:
@@ -14,22 +16,20 @@ class CertificateId:
     raw: str
 
     def _canonical_fr(self) -> str:
-        def pad_last_segment_with_zero(id_str: str) -> str:
-            splitted = id_str.split("/")
-            if len(splitted) > 1:
-                num = splitted[-1].zfill(2)
-                return f"{''.join(splitted[:-1])}/{num}"
-            return id_str
-
         new_cert_id = self.clean
-        rules = [
-            "(?:Rapport de certification|Certification Report) ([0-9]+[/-_][0-9]+(?:[vV][1-9])?(?:[_/-][MSR][0-9]+)?)",
-            "(?:ANSS[Ii]|DCSSI)(?:-CC)?[- ]([0-9]+[/-_][0-9]+(?:[vV][1-9])?(?:[_/-][MSR][0-9]+)?)",
-            "([0-9]+[/-_][0-9]+(?:[vV][1-9])?(?:[_/-][MSR][0-9]+)?)",
-        ]
-        for rule in rules:
+        for rule in rules["cc_cert_id"]["FR"]:
             if match := re.match(rule, new_cert_id):
-                return pad_last_segment_with_zero("ANSSI-CC-" + match.group(1).replace("_", "/").replace("V", "v"))
+                groups = match.groupdict()
+                year = int(groups["year"]) if len(groups["year"]) == 4 else int(groups["year"]) + 2000
+                counter = groups["counter"]
+                doc = groups.get("doc")
+                version = groups.get("version")
+                new_cert_id = f"ANSSI-CC-{year}/{counter}"
+                if doc:
+                    new_cert_id += f"-{doc}"
+                if version:
+                    new_cert_id += f"v{version}"
+                return new_cert_id
 
         return new_cert_id
 
-- 
cgit v1.3.1


From 67861ffd16f215f7d476e90bb8e480a5c60dc01b Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 00:11:20 +0100
Subject: Improve BSI regexes.

---
 src/sec_certs/rules.yaml                  | 20 +++++-----
 src/sec_certs/sample/cc_certificate_id.py | 61 ++++++++++++++++---------------
 tests/cc/test_cc_misc.py                  |  3 ++
 3 files changed, 45 insertions(+), 39 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 87acd04b..5604b709 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -5,17 +5,17 @@
 #####
 cc_cert_id:
   DE:
-    - "BSI-DSZ-CC-[0-9]+?-[0-9]+"
-    - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+-[0-9]+"
-    - "BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+"
-    - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(-[0-9]{4})*"  # German BSI (number + version + year or without year)
-    - "BSI-DSZ-CC-[0-9]+-[0-9]{4}"  # German BSI (number + year, no version)
-    - "BSI-DSZ-CC-[0-9]+-(?:V|v)[0-9]+(?!-)"  # German BSI (number + version but no year => no - after version)
-    # - "BSI-DSZ-CC-[0-9]+"  # Maybe?
+    - "BSI-DSZ-CC-(?:(?P<s>S)-)?(?P<counter>[0-9]{3,5})-(?:(?P<version>[vV][0-9])-)?(?P<year>[0-9]{4})?(?:-(?P<doc>(?:RA|MA)(?:-[0-9]+)?))?"
+    # Examples:
+    # BSI-DSZ-CC-1004
+    # BSI-DSZ-CC-0973-2016
+    # BSI-DSZ-CC-0831-V4-2021
+    # BSI-DSZ-CC-0837-V2-2014-MA-01
+    # BSI-DSZ-CC-S-0192-2021
   FR:
     - "DCSS[Ii]-(?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)([vV](?P<version>[0-9]))?"
-    - "Certification Report (?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)([vV](?P<version>[0-9]))?"
     - "Rapport de certification (?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)([vV](?P<version>[0-9]))?"
+    - "Certification Report (?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)([vV](?P<version>[0-9]))?"
     - "ANSS[Ii](?:-CC)?[ -](?P<year>[0-9]{2,4})[/_-](?P<counter>[0-9]+)(?:-(?P<doc>(?:[MSR][0-9]+)))?([vV](?P<version>[0-9]))?"
     # Examples:
     # DCSSI-2009/07
@@ -30,7 +30,9 @@ cc_cert_id:
     # CC-16-31801-CR4  (no NSCIB)
     # NSCIB-CC-98209   (no year, no CR)
   "NO":
-    - "SERTIT-(?P<counter>[0-9]+)"  # Norway
+    - "SERTIT-(?P<counter>[0-9]+)"
+    # Examples:
+    # SERTIT-101
   US:
     - "CCEVS-VR-(?:CC-|VID)?[0-9]+-[0-9]+[a-z]?(?:-[0-9]+)?"  # US NSA (CCEVS-VR-10884-2018 CCEVS-VR-VID10877-2018)
   CA:
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 9bce0764..188a2163 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -6,6 +6,16 @@ from dataclasses import dataclass
 from sec_certs.cert_rules import rules
 
 
+def _parse_year(year: str | None) -> int | None:
+    if year is None:
+        return None
+    y = int(year)
+    if y < 100:
+        return y + 1900
+    else:
+        return y
+
+
 @dataclass(eq=True, frozen=True)
 class CertificateId:
     """
@@ -20,7 +30,7 @@ class CertificateId:
         for rule in rules["cc_cert_id"]["FR"]:
             if match := re.match(rule, new_cert_id):
                 groups = match.groupdict()
-                year = int(groups["year"]) if len(groups["year"]) == 4 else int(groups["year"]) + 2000
+                year = _parse_year(groups["year"])
                 counter = groups["counter"]
                 doc = groups.get("doc")
                 version = groups.get("version")
@@ -34,35 +44,26 @@ class CertificateId:
         return new_cert_id
 
     def _canonical_de(self) -> str:
-        def extract_parts(bsi_parts: list[str]) -> tuple:
-            cert_num = None
-            cert_version = None
-            cert_year = None
-
-            if len(bsi_parts) > 3:
-                cert_num = bsi_parts[3]
-            if len(bsi_parts) > 4:
-                if bsi_parts[4].startswith("V") or bsi_parts[4].startswith("v"):
-                    cert_version = bsi_parts[4].upper()  # get version in uppercase
-                else:
-                    cert_year = bsi_parts[4]
-            if len(bsi_parts) > 5:
-                cert_year = bsi_parts[5]
-
-            return cert_num, cert_version, cert_year
-
-        bsi_parts = self.clean.split("-")
-
-        cert_num, cert_version, cert_year = extract_parts(bsi_parts)
-
-        # reconstruct BSI number again
-        new_cert_id = "BSI-DSZ-CC"
-        if cert_num is not None:
-            new_cert_id += "-" + cert_num
-        if cert_version is not None:
-            new_cert_id += "-" + cert_version
-        if cert_year is not None:
-            new_cert_id += "-" + cert_year
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["DE"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                s = groups.get("s")
+                counter = groups["counter"]
+                version = groups.get("version")
+                year = _parse_year(groups.get("year"))
+                doc = groups.get("doc")
+                new_cert_id = "BSI-DSZ-CC"
+                if s:
+                    new_cert_id += f"-{s}"
+                new_cert_id += f"-{counter}"
+                if version:
+                    new_cert_id += f"-{version.upper()}"
+                if year:
+                    new_cert_id += f"-{year}"
+                if doc:
+                    new_cert_id += f"-{doc}"
+                return new_cert_id
 
         return new_cert_id
 
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index ae9a83aa..b9654c91 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -8,6 +8,9 @@ def test_canonicalize_fr():
 
 def test_canonicalize_de():
     assert canonicalize("BSI-DSZ-CC-0420-2007", "DE") == "BSI-DSZ-CC-0420-2007"
+    assert canonicalize("BSI-DSZ-CC-1004", "DE") == "BSI-DSZ-CC-1004"
+    assert canonicalize("BSI-DSZ-CC-0831-V4-2021", "DE") == "BSI-DSZ-CC-0831-V4-2021"
+    assert canonicalize("BSI-DSZ-CC-0837-V2-2014-MA-01", "DE") == "BSI-DSZ-CC-0837-V2-2014-MA-01"
 
 
 def test_canonicalize_es():
-- 
cgit v1.3.1


From c60fece36b7e7c5300b25cf83b9e34d338675067 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 15:41:11 +0100
Subject: Improve US scheme rules.

---
 src/sec_certs/rules.yaml                  |  7 ++++++-
 src/sec_certs/sample/cc_certificate_id.py | 29 ++++++++++++++++++++++++++---
 tests/cc/test_cc_misc.py                  |  6 ++++++
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 5604b709..fa188052 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -34,7 +34,12 @@ cc_cert_id:
     # Examples:
     # SERTIT-101
   US:
-    - "CCEVS-VR-(?:CC-|VID)?[0-9]+-[0-9]+[a-z]?(?:-[0-9]+)?"  # US NSA (CCEVS-VR-10884-2018 CCEVS-VR-VID10877-2018)
+    - "CCEVS-VR-(?:(?P<cc>CC)-)?(?:(?P<VID>VID)-?)?(?P<year>[0-9]{2})-(?P<counter>[0-9]+)"
+    - "CCEVS-VR-(?:(?P<cc>CC)-)?(?:(?P<VID>VID)-?)?(?P<counter>[0-9]{4,5})-(?P<year>[0-9]{4})?"
+    # Examples:
+    # CCEVS-VR-VID10015-2008
+    # CCEVS-VR-10880-2018
+    # CCEVS-VR-04-0082
   CA:
     # '[0-9][0-9\-]+?-CR', # Canada
     - "[0-9][0-9][0-9]-[347]-[0-9][0-9][0-9]?(?:-CR|P)?"  # Canada xxx-{347}-xxx (383-4-438, 383-4-82-CR, 383-4-422P)
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 188a2163..e84f6df2 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -10,7 +10,9 @@ def _parse_year(year: str | None) -> int | None:
     if year is None:
         return None
     y = int(year)
-    if y < 100:
+    if y < 50:
+        return y + 2000
+    elif y < 100:
         return y + 1900
     else:
         return y
@@ -39,7 +41,7 @@ class CertificateId:
                     new_cert_id += f"-{doc}"
                 if version:
                     new_cert_id += f"v{version}"
-                return new_cert_id
+                break
 
         return new_cert_id
 
@@ -63,7 +65,27 @@ class CertificateId:
                     new_cert_id += f"-{year}"
                 if doc:
                     new_cert_id += f"-{doc}"
-                return new_cert_id
+                break
+
+        return new_cert_id
+
+    def _canonical_us(self) -> str:
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["US"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                year = _parse_year(groups["year"])
+                counter = groups["counter"]
+                cc = groups.get("cc")
+                vid = groups.get("VID")
+                new_cert_id = "CCEVS-VR"
+                if cc:
+                    new_cert_id += f"-{cc}"
+                if vid:
+                    new_cert_id += f"-{vid}"
+                new_cert_id += f"-{counter}"
+                new_cert_id += f"-{year}"
+                break
 
         return new_cert_id
 
@@ -147,6 +169,7 @@ class CertificateId:
         schemes = {
             "FR": self._canonical_fr,
             "DE": self._canonical_de,
+            "US": self._canonical_us,
             "ES": self._canonical_es,
             "IT": self._canonical_it,
             "IN": self._canonical_in,
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index b9654c91..6fde3339 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -13,6 +13,12 @@ def test_canonicalize_de():
     assert canonicalize("BSI-DSZ-CC-0837-V2-2014-MA-01", "DE") == "BSI-DSZ-CC-0837-V2-2014-MA-01"
 
 
+def test_canonicalize_us():
+    assert canonicalize("CCEVS-VR-VID10015-2008", "US") == "CCEVS-VR-VID-10015-2008"
+    assert canonicalize("CCEVS-VR-10880-2018", "US") == "CCEVS-VR-10880-2018"
+    assert canonicalize("CCEVS-VR-04-0082", "US") == "CCEVS-VR-0082-2004"
+
+
 def test_canonicalize_es():
     assert canonicalize("2011-14-INF-1095-v1", "ES") == "2011-14-INF-1095"
 
-- 
cgit v1.3.1


From 48876d5979adf8a63e2141262acd16608fc2aeda Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 17:14:17 +0100
Subject: Improve Malaysian rules.

---
 src/sec_certs/rules.yaml                  |  6 +++++-
 src/sec_certs/sample/cc_certificate_id.py | 14 ++++++++++++++
 tests/cc/test_cc_misc.py                  |  6 ++++++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index fa188052..6b63b991 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -60,7 +60,11 @@ cc_cert_id:
     - "JISEC-CC-CRP-C[0-9]+-[0-9]+-[0-9]+"  # Japan (JISEC-CC-CRP-C0689-01-2020)
     - "Certification No. [cC][0-9]+" # Japan (Certification No. C0090)
   MY:
-    - "ISCB-[0-9]+-(?:RPT|FRM)-[CM][0-9]+[A-Z]?-(?:CR|AMR)(?:-[0-9])?-[vV][0-9](?:\\.[0-9])?[a-z]?" # Malaysia (ISCB-3-RPT-C092-CR-v1, ISCB-3-RPT-C068-CR-1-v1)
+    - "ISCB-(?P<digit>[0-9])-RPT-C(?P<counter>[0-9]{3})-CR(?:-[0-9])?-(?P<version>[vV][0-9][a-z]?)"
+    # Examples:
+    # ISCB-3-RPT-C068-CR-1-v1
+    # ISCB-5-RPT-C075-CR-v2
+    # ISCB-5-RPT-C046-CR-V1a
   IT:
     - "OCSI/CERT/.+?"  # Italy
     - "OCSI/CERT/.+?/20[0-9]+(?:\\w|/RC)"  # Italy  (OCSI/CERT/ATS/01/2018/RC)
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index e84f6df2..42dc9e6f 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -89,6 +89,19 @@ class CertificateId:
 
         return new_cert_id
 
+    def _canonical_my(self) -> str:
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["MY"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                digit = groups["digit"]
+                counter = groups["counter"]
+                version = groups["version"]
+                new_cert_id = f"ISCB-{digit}-RPT-C{counter}-CR-{version.lower()}"
+                break
+
+        return new_cert_id
+
     def _canonical_es(self) -> str:
         cert_id = self.clean
         spain_parts = cert_id.split("-")
@@ -170,6 +183,7 @@ class CertificateId:
             "FR": self._canonical_fr,
             "DE": self._canonical_de,
             "US": self._canonical_us,
+            "MY": self._canonical_my,
             "ES": self._canonical_es,
             "IT": self._canonical_it,
             "IN": self._canonical_in,
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index 6fde3339..d439d508 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -19,6 +19,12 @@ def test_canonicalize_us():
     assert canonicalize("CCEVS-VR-04-0082", "US") == "CCEVS-VR-0082-2004"
 
 
+def test_canonicalize_my():
+    assert canonicalize("ISCB-5-RPT-C075-CR-v2", "MY") == "ISCB-5-RPT-C075-CR-v2"
+    assert canonicalize("ISCB-5-RPT-C046-CR-V1a", "MY") == "ISCB-5-RPT-C046-CR-v1a"
+    assert canonicalize("ISCB-3-RPT-C068-CR-1-v1", "MY") == "ISCB-3-RPT-C068-CR-v1"
+
+
 def test_canonicalize_es():
     assert canonicalize("2011-14-INF-1095-v1", "ES") == "2011-14-INF-1095"
 
-- 
cgit v1.3.1


From affffb9fc08b09d39806815a741832df79cd95e5 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 18:37:48 +0100
Subject: Improve Japanese rules.

---
 src/sec_certs/rules.yaml                  | 10 +++++++---
 src/sec_certs/sample/cc_certificate_id.py | 16 ++++++++++++----
 tests/cc/test_cc_misc.py                  |  6 +++---
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 6b63b991..95a834ba 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -56,9 +56,13 @@ cc_cert_id:
     # - "KECS[-‐]CR[-‐][0-9]+[-‐][0-9]+"  # Korea KECS-CR-20-61
     - "KECS[-‐](?:ISIS|NISS|CISS)[-‐][0-9]+[-‐][0-9]{4}"  # Korea KECS-ISIS-1234-2011
   JP:
-    - "(?:CRP|ACR)-C[0-9]+-[0-9]+"  # Japan (CRP-C0595-01 ACR-C0417-03)
-    - "JISEC-CC-CRP-C[0-9]+-[0-9]+-[0-9]+"  # Japan (JISEC-CC-CRP-C0689-01-2020)
-    - "Certification No. [cC][0-9]+" # Japan (Certification No. C0090)
+    - "(?:CRP|ACR)-C(?P<counter>[0-9]+)-(?P<digit>[0-9]+)"
+    - "JISEC-CC-CRP-C(?P<counter>[0-9]+)-(?P<digit>[0-9]+)-(?P<year>[0-9]{4})"
+    - "Certification No. [cC](?P<counter>[0-9]+)"
+    # Examples:
+    # CRP-C0595-01
+    # JISEC-CC-CRP-C0689-01-2020
+    # Certification No. C0090
   MY:
     - "ISCB-(?P<digit>[0-9])-RPT-C(?P<counter>[0-9]{3})-CR(?:-[0-9])?-(?P<version>[vV][0-9][a-z]?)"
     # Examples:
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 42dc9e6f..fe362b99 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -147,10 +147,18 @@ class CertificateId:
 
     def _canonical_jp(self):
         new_cert_id = self.clean
-        if match := re.match("Certification No. (C[0-9]+)", new_cert_id):
-            return match.group(1)
-        if match := re.search("CRP-(C[0-9]+)-", new_cert_id):
-            return match.group(1)
+        for rule in rules["cc_cert_id"]["JP"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                counter = groups["counter"]
+                digit = groups.get("digit")
+                year = _parse_year(groups.get("year"))
+                new_cert_id = f"JISEC-CC-CRP-C{counter}"
+                if digit:
+                    new_cert_id += f"-{digit}"
+                if year:
+                    new_cert_id += f"-{year}"
+                break
         return new_cert_id
 
     def _canonical_no(self):
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index d439d508..3772340f 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -53,9 +53,9 @@ def test_canonicalize_ca():
 
 
 def test_canonicalize_jp():
-    assert canonicalize("Certification No. C01234", "JP") == "C01234"
-    assert canonicalize("CRP-C01234-01", "JP") == "C01234"
-    assert canonicalize("JISEC-CC-CRP-C0689-01-2020", "JP") == "C0689"
+    assert canonicalize("Certification No. C01234", "JP") == "JISEC-CC-CRP-C01234"
+    assert canonicalize("CRP-C01234-01", "JP") == "JISEC-CC-CRP-C01234-01"
+    assert canonicalize("JISEC-CC-CRP-C0689-01-2020", "JP") == "JISEC-CC-CRP-C0689-01-2020"
 
 
 def test_canonicalize_no():
-- 
cgit v1.3.1


From a096f7ed6c4637dc700bd53df2d110bb8a428394 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 18:45:46 +0100
Subject: Improve UK rules.

---
 src/sec_certs/rules.yaml                  | 7 +++++--
 src/sec_certs/sample/cc_certificate_id.py | 8 ++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 95a834ba..8f5c6fd4 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -46,8 +46,11 @@ cc_cert_id:
     - "[0-9][0-9][0-9][ -](?:EWA|LSS|CCS)(?:[ -]20[0-9][0-9])?"  # Canada (522-EWA-2020, 524 LSS 2020, 503-LSS)
     - "[0-9][0-9][0-9](?:%20|-)(?:EWA|LSS|CCS)(?:%20|-)(?:20[0-9][0-9]%20|)CR%20v[0-9]\\.[0-9]" # Canada filename with space (518-LSS%20CR%20v1.0)
   UK:
-    - "CRP[0-9]+[A-Z]?"  # UK CESG
-    - "CERTIFICATION REPORT No. P[0-9]+[A-Z]?"  # UK CESG
+    - "CRP(?P<counter>[0-9]+[A-Z]?)"
+    - "CERTIFICATION REPORT No. P(?P<counter>[0-9]+[A-Z]?)"
+    # Examples:
+    # CRP208
+    # CERTIFICATION REPORT No. P123A
   ES:
     - "20[0-9][0-9][-‐][0-9]+[-‐]INF[-‐][0-9]+([-‐]?[ -‐](?:V|v)[0-9]+)?"  # Spain ("2006-4-INF-98 v2" or "2006-4-INF-98-v2" or "2020-34-INF-3784- v1")
   KR:
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index fe362b99..5ec168ee 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -133,8 +133,12 @@ class CertificateId:
 
     def _canonical_uk(self):
         new_cert_id = self.clean
-        if match := re.match("CERTIFICATION REPORT No. P([0-9]+[A-Z]?)", new_cert_id):
-            new_cert_id = "CRP" + match.group(1)
+        for rule in rules["cc_cert_id"]["UK"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                counter = groups["counter"]
+                new_cert_id = f"CRP{counter}"
+                break
         return new_cert_id
 
     def _canonical_ca(self):
-- 
cgit v1.3.1


From 3efe520d01726c5c00428b9fd4b4d07d8bb4da5c Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 18:52:17 +0100
Subject: Improve Swedish rules.

---
 src/sec_certs/rules.yaml                  | 5 ++++-
 src/sec_certs/sample/cc_certificate_id.py | 9 ++++++++-
 tests/cc/test_cc_misc.py                  | 2 ++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 8f5c6fd4..404931fc 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -79,7 +79,10 @@ cc_cert_id:
     - "[0-9\\.]+?/TSE-CCCS-[0-9]+"  # Turkish CCCS (21.0.0sc/TSE-CCCS-75)
     - "(?:[0-9]{1,2}\\.){2}[0-9]{1,2}/[0-9]{1,4}-[0-9]{3}"   # 21.0.01/13-028
   SE:
-    - "CSEC ?[0-9]{6,7}"  # Sweden (CSEC2019015)
+    - "CSEC ?(?P<year>[0-9]{4})(?P<counter>[0-9]{2,3})"
+    # Examples:
+    # CSEC2019015
+    # CSEC 2019012
   IN:
     # India (IC3S/DEL01/VALIANT/EAL1/0317/0007/CR  STQC/CC/14-15/12/ETR/0017 IC3S/MUM01/CISCO/cPP/0119/0016/CR)
     # will miss STQC/CC/14-15/12/ETR/0017
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 5ec168ee..bdca0ab6 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -129,7 +129,14 @@ class CertificateId:
         return self.clean.replace(" ", "")
 
     def _canonical_se(self):
-        return self.clean.replace(" ", "")
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["SE"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                year = _parse_year(groups["year"])
+                counter = int(groups["counter"])
+                new_cert_id = f"CSEC{year}{counter:03}"
+        return new_cert_id
 
     def _canonical_uk(self):
         new_cert_id = self.clean
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index 3772340f..b25620e4 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -4,6 +4,7 @@ from sec_certs.sample.cc_certificate_id import canonicalize
 def test_canonicalize_fr():
     assert canonicalize("Rapport de certification 2001/02v2", "FR") == "ANSSI-CC-2001/02v2"
     assert canonicalize("ANSSI-CC 2001/02-R01", "FR") == "ANSSI-CC-2001/02-R01"
+    assert canonicalize("ANSSI-CC 2001_02-M01", "FR") == "ANSSI-CC-2001/02-M01"
 
 
 def test_canonicalize_de():
@@ -40,6 +41,7 @@ def test_canonicalize_in():
 def test_canonicalize_se():
     assert canonicalize("CSEC2017020", "SE") == "CSEC2017020"
     assert canonicalize("CSEC 2017020", "SE") == "CSEC2017020"
+    assert canonicalize("CSEC201003", "SE") == "CSEC2010003"
 
 
 def test_canonicalize_uk():
-- 
cgit v1.3.1


From 22fad93ad165e858d4fb55e2b9b0166aac6d32a3 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 19:09:54 +0100
Subject: Improve Spanish rules.

---
 src/sec_certs/rules.yaml                  |  6 +++++-
 src/sec_certs/sample/cc_certificate_id.py | 24 +++++++++++-------------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 404931fc..b4e54d20 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -52,7 +52,11 @@ cc_cert_id:
     # CRP208
     # CERTIFICATION REPORT No. P123A
   ES:
-    - "20[0-9][0-9][-‐][0-9]+[-‐]INF[-‐][0-9]+([-‐]?[ -‐](?:V|v)[0-9]+)?"  # Spain ("2006-4-INF-98 v2" or "2006-4-INF-98-v2" or "2020-34-INF-3784- v1")
+    - "(?P<year>[0-9]{4})[-‐](?P<project>[0-9]+)[-‐]INF[-‐](?P<counter>[0-9]+)[ -‐]{1,2}[vV](?P<version>[0-9])"
+    # Examples:
+    # 2006-4-INF-98 v2
+    # 2020-34-INF-3784- v1
+    # 2019-20-INF-3379-v1
   KR:
     # Korea
     # XXX: Do not use KECS-CR as those refer to the certificate report and do not represent the certificate id.
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index bdca0ab6..be30bcf5 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -103,19 +103,17 @@ class CertificateId:
         return new_cert_id
 
     def _canonical_es(self) -> str:
-        cert_id = self.clean
-        spain_parts = cert_id.split("-")
-        cert_year = spain_parts[0]
-        cert_batch = spain_parts[1].lstrip("0")
-        cert_num = spain_parts[3].lstrip("0")
-
-        if "v" in cert_num:
-            cert_num = cert_num[: cert_num.find("v")]
-        if "V" in cert_num:
-            cert_num = cert_num[: cert_num.find("V")]
-
-        new_cert_id = f"{cert_year}-{cert_batch}-INF-{cert_num.strip()}"  # drop version # TODO: Maybe do not drop?
-
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["ES"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                year = _parse_year(groups["year"])
+                project = groups["project"]
+                counter = groups["counter"]
+                # Version is intentionally cut here, as it seems to refer to an internal version of the report.
+                # version = groups["version"]
+                new_cert_id = f"{year}-{project}-INF-{counter}"
+                break
         return new_cert_id
 
     def _canonical_it(self):
-- 
cgit v1.3.1


From 7d56381b408dd6ab1ed7d7ebaa84d192419eb140 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 19:27:40 +0100
Subject: Improve Singaporean and Australian rules.

---
 src/sec_certs/rules.yaml                  | 14 +++++++++-----
 src/sec_certs/sample/cc_certificate_id.py | 17 +++++++++++++++++
 tests/cc/test_cc_misc.py                  |  7 +++++++
 3 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index b4e54d20..b2b4a979 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -92,13 +92,17 @@ cc_cert_id:
     # will miss STQC/CC/14-15/12/ETR/0017
     - "(?:IC3S|STQC/CC)/[^ ]+? ?/CR"
   SG:
-    - "CSA_CC_[0-9]+"  # Singapore (CSA_CC_19001)
+    - "CSA_CC_(?P<year>[0-9]{2})(?P<counter>[0-9]{3})"
+    # Examples:
+    # CSA_CC_19001
   AU:
-    # Australia (EFS-T048 ETR 1.0, EFS-T056-ETR 1.0, DXC-EFC-T092-ETR 1.0)
+    - "(?:Certificate Number:|Certification Report) (?P<year>[0-9]{2,4})/(?P<counter>[0-9]+)"
     # XXX: Do not use Australian ETR numbers, they are not certificate id.
-    # - "(?:EFS|EFT|DXC-EFC)-T[0-9]+(?: |-)ETR [0-9]+.[0-9]+"
-    - "Certificate Number: [0-9]{1,4}/[0-9]{1,4}"
-    - "Certification Report [0-9]+/[0-9]+"
+    # Examples:
+    # Certification Report 2007/06
+    # Certificate Number: 2010/67
+    # Certificate Number: 37/2006    !mistake
+    # Certification Report 97/76     !short year
 
 #####
 # Common Criteria protection profile IDs, grouped by certification body (e.g. BSI)
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index be30bcf5..7e0b7f4a 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -183,6 +183,21 @@ class CertificateId:
             new_cert_id = f"{new_cert_id}-CR"
         return new_cert_id
 
+    def _canonical_au(self):
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["AU"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                counter = groups["counter"]
+                year_s = groups["year"]
+                if len(year_s) < len(counter):
+                    # Hack for some mistakes in their ordering
+                    year_s, counter = counter, year_s
+                year = _parse_year(year_s)
+                new_cert_id = f"Certificate Number: {year}/{counter}"
+                break
+        return new_cert_id
+
     @property
     def clean(self) -> str:
         """
@@ -210,6 +225,8 @@ class CertificateId:
             "JP": self._canonical_jp,
             "NO": self._canonical_no,
             "NL": self._canonical_nl,
+            "AU": self._canonical_au,
+            # SG is canonical by default
         }
 
         if self.scheme in schemes:
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index b25620e4..e1d516b3 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -49,6 +49,13 @@ def test_canonicalize_uk():
     assert canonicalize("CRP123A", "UK") == "CRP123A"
 
 
+def test_canonicalize_au():
+    assert canonicalize("Certification Report 2007/02", "AU") == "Certificate Number: 2007/02"
+    assert canonicalize("Certificate Number: 37/2006", "AU") == "Certificate Number: 2006/37"
+    assert canonicalize("Certificate Number: 2011/73", "AU") == "Certificate Number: 2011/73"
+    assert canonicalize("Certification Report 97/76", "AU") == "Certificate Number: 1997/76"
+
+
 def test_canonicalize_ca():
     assert canonicalize("383-4-123-CR", "CA") == "383-4-123"
     assert canonicalize("383-4-123P", "CA") == "383-4-123"
-- 
cgit v1.3.1


From 5e852ac4e42afaf7fb7a6f7b9493d276723fd7e7 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Sat, 3 Feb 2024 19:36:05 +0100
Subject: Improve Italian rules.

---
 src/sec_certs/rules.yaml                  | 8 ++++++--
 src/sec_certs/sample/cc_certificate_id.py | 9 +--------
 tests/cc/test_cc_misc.py                  | 4 ----
 3 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index b2b4a979..c0acd3d4 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -77,8 +77,12 @@ cc_cert_id:
     # ISCB-5-RPT-C075-CR-v2
     # ISCB-5-RPT-C046-CR-V1a
   IT:
-    - "OCSI/CERT/.+?"  # Italy
-    - "OCSI/CERT/.+?/20[0-9]+(?:\\w|/RC)"  # Italy  (OCSI/CERT/ATS/01/2018/RC)
+    - "OCSI/CERT/(?:(?P<lab>[A-Z]{3})/)?(?P<counter>[0-9]{2,3})/(?P<year>[0-9]{4})/RC"
+    # Examples:
+    # OCSI/CERT/SYS/04/2018/RC
+    # OCSI/CERT/CCL/10/2022/RC
+    # OCSI/CERT/TEC/09/2017/RC
+    # OCSI/CERT/ATS/06/2020/RC
   TR:
     - "[0-9\\.]+?/TSE-CCCS-[0-9]+"  # Turkish CCCS (21.0.0sc/TSE-CCCS-75)
     - "(?:[0-9]{1,2}\\.){2}[0-9]{1,2}/[0-9]{1,4}-[0-9]{3}"   # 21.0.01/13-028
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 7e0b7f4a..88e1f4a7 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -116,13 +116,6 @@ class CertificateId:
                 break
         return new_cert_id
 
-    def _canonical_it(self):
-        new_cert_id = self.clean
-        if not new_cert_id.endswith("/RC"):
-            new_cert_id = new_cert_id + "/RC"
-
-        return new_cert_id
-
     def _canonical_in(self):
         return self.clean.replace(" ", "")
 
@@ -217,7 +210,6 @@ class CertificateId:
             "US": self._canonical_us,
             "MY": self._canonical_my,
             "ES": self._canonical_es,
-            "IT": self._canonical_it,
             "IN": self._canonical_in,
             "SE": self._canonical_se,
             "UK": self._canonical_uk,
@@ -227,6 +219,7 @@ class CertificateId:
             "NL": self._canonical_nl,
             "AU": self._canonical_au,
             # SG is canonical by default
+            # IT is canonucal by default
         }
 
         if self.scheme in schemes:
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index e1d516b3..2c93c546 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -30,10 +30,6 @@ def test_canonicalize_es():
     assert canonicalize("2011-14-INF-1095-v1", "ES") == "2011-14-INF-1095"
 
 
-def test_canonicalize_it():
-    assert canonicalize("OCSI/CERT/SYS/10/2016", "IT") == "OCSI/CERT/SYS/10/2016/RC"
-
-
 def test_canonicalize_in():
     assert canonicalize("IC3S/KOL01/ADVA/EAL2/0520/0021 /CR", "IN") == "IC3S/KOL01/ADVA/EAL2/0520/0021/CR"
 
-- 
cgit v1.3.1


From 4fbb063846eb2375ac735c88ca0abb7df7c11267 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 13:50:58 +0100
Subject: Improve Korean rules.

---
 src/sec_certs/rules.yaml                  |  8 +++++---
 src/sec_certs/sample/cc_certificate_id.py | 13 +++++++++++++
 tests/cc/test_cc_misc.py                  |  5 +++++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index c0acd3d4..5e000144 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -58,10 +58,12 @@ cc_cert_id:
     # 2020-34-INF-3784- v1
     # 2019-20-INF-3379-v1
   KR:
-    # Korea
+    - "KECS[-‐](?P<word>ISIS|NISS|CISS)[-‐](?P<counter>[0-9]{2,4})[-‐](?P<year>[0-9]{4})"
     # XXX: Do not use KECS-CR as those refer to the certificate report and do not represent the certificate id.
-    # - "KECS[-‐]CR[-‐][0-9]+[-‐][0-9]+"  # Korea KECS-CR-20-61
-    - "KECS[-‐](?:ISIS|NISS|CISS)[-‐][0-9]+[-‐][0-9]{4}"  # Korea KECS-ISIS-1234-2011
+    # Examples:
+    # KECS-ISIS-0579-2015
+    # KECS-NISS-0792-2017
+    # KECS-CISS-1210-2023
   JP:
     - "(?:CRP|ACR)-C(?P<counter>[0-9]+)-(?P<digit>[0-9]+)"
     - "JISEC-CC-CRP-C(?P<counter>[0-9]+)-(?P<digit>[0-9]+)-(?P<year>[0-9]{4})"
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 88e1f4a7..5facf1a5 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -163,6 +163,18 @@ class CertificateId:
                 break
         return new_cert_id
 
+    def _canonical_kr(self):
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["KR"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                word = groups["word"]
+                counter = int(groups["counter"])
+                year = _parse_year(groups["year"])
+                new_cert_id = f"KECS-{word}-{counter:04}-{year}"
+                break
+        return new_cert_id
+
     def _canonical_no(self):
         new_cert_id = self.clean
         cert_num = int(new_cert_id.split("-")[1])
@@ -218,6 +230,7 @@ class CertificateId:
             "NO": self._canonical_no,
             "NL": self._canonical_nl,
             "AU": self._canonical_au,
+            "KR": self._canonical_kr,
             # SG is canonical by default
             # IT is canonucal by default
         }
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index 2c93c546..50bd81c0 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -63,6 +63,11 @@ def test_canonicalize_jp():
     assert canonicalize("JISEC-CC-CRP-C0689-01-2020", "JP") == "JISEC-CC-CRP-C0689-01-2020"
 
 
+def test_canonicalize_KR():
+    assert canonicalize("KECS-ISIS-0579-2015", "KR") == "KECS-ISIS-0579-2015"
+    assert canonicalize("KECS-CISS-10-2023", "KR") == "KECS-CISS-0010-2023"
+
+
 def test_canonicalize_no():
     assert canonicalize("SERTIT-12", "NO") == "SERTIT-012"
 
-- 
cgit v1.3.1


From 520ce49b95c7f79481a68f23ec74bffbc93a80c1 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 14:05:28 +0100
Subject: Improve Indian rules.

---
 src/sec_certs/rules.yaml                  | 10 +++++++---
 src/sec_certs/sample/cc_certificate_id.py | 14 ++++++++++++--
 tests/cc/test_cc_misc.py                  |  2 +-
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 5e000144..41357015 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -94,9 +94,13 @@ cc_cert_id:
     # CSEC2019015
     # CSEC 2019012
   IN:
-    # India (IC3S/DEL01/VALIANT/EAL1/0317/0007/CR  STQC/CC/14-15/12/ETR/0017 IC3S/MUM01/CISCO/cPP/0119/0016/CR)
-    # will miss STQC/CC/14-15/12/ETR/0017
-    - "(?:IC3S|STQC/CC)/[^ ]+? ?/CR"
+    - "IC3S/(?P<lab>[A-Z]+[0-9]+)/(?P<vendor>[a-zA-Z_]+)/(?P<level>[a-zA-Z0-9]+)/(?P<number1>[0-9]+)/(?P<number2>[0-9]+) ?(?:/CR)?"
+    # XXX: The cert IDs are often present only in the certificate and not in the report.
+    #      The report often only has the report id, of the format "STQC/CC/1617/18/CR"
+    # Examples:
+    # IC3S/BG01/HALTDOS/EAL2/0317/0008/CR
+    # IC3S/KOL01/ADVA/EAL2/0520/0021/CR
+    # IC3S/MUM01/Symantec/NDcPP/0722/0032/CR
   SG:
     - "CSA_CC_(?P<year>[0-9]{2})(?P<counter>[0-9]{3})"
     # Examples:
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 5facf1a5..7997b846 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -117,7 +117,17 @@ class CertificateId:
         return new_cert_id
 
     def _canonical_in(self):
-        return self.clean.replace(" ", "")
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["IN"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                lab = groups["lab"]
+                vendor = groups["vendor"]
+                level = groups["level"]
+                number1, number2 = groups["number1"], groups["number2"]
+                new_cert_id = f"IC3S/{lab}/{vendor}/{level}/{number1}/{number2}"
+                break
+        return new_cert_id
 
     def _canonical_se(self):
         new_cert_id = self.clean
@@ -232,7 +242,7 @@ class CertificateId:
             "AU": self._canonical_au,
             "KR": self._canonical_kr,
             # SG is canonical by default
-            # IT is canonucal by default
+            # IT is canonical by default
         }
 
         if self.scheme in schemes:
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index 50bd81c0..ca32fe91 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -31,7 +31,7 @@ def test_canonicalize_es():
 
 
 def test_canonicalize_in():
-    assert canonicalize("IC3S/KOL01/ADVA/EAL2/0520/0021 /CR", "IN") == "IC3S/KOL01/ADVA/EAL2/0520/0021/CR"
+    assert canonicalize("IC3S/KOL01/ADVA/EAL2/0520/0021 /CR", "IN") == "IC3S/KOL01/ADVA/EAL2/0520/0021"
 
 
 def test_canonicalize_se():
-- 
cgit v1.3.1


From 1927e44223723c038c98ee0516bf67f6a20eff8e Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 15:28:41 +0100
Subject: Improve Turkish rules.

---
 src/sec_certs/rules.yaml                  |  7 +++++--
 src/sec_certs/sample/cc_certificate_id.py | 12 ++++++++++++
 tests/cc/test_cc_misc.py                  |  7 ++++++-
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 41357015..3cb5d4d3 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -86,8 +86,11 @@ cc_cert_id:
     # OCSI/CERT/TEC/09/2017/RC
     # OCSI/CERT/ATS/06/2020/RC
   TR:
-    - "[0-9\\.]+?/TSE-CCCS-[0-9]+"  # Turkish CCCS (21.0.0sc/TSE-CCCS-75)
-    - "(?:[0-9]{1,2}\\.){2}[0-9]{1,2}/[0-9]{1,4}-[0-9]{3}"   # 21.0.01/13-028
+    - "(?P<prefix>[0-9\\.]+)/TSE-CCCS-(?P<number>[0-9]+)"
+    # XXX: The report numbers are like "21.0.01/13-028"
+    # Examples:
+    # 21.0.03.0.00.00/TSE-CCCS-85
+    # 21.0.03/TSE-CCCS-33
   SE:
     - "CSEC ?(?P<year>[0-9]{4})(?P<counter>[0-9]{2,3})"
     # Examples:
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 7997b846..901629c7 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -185,6 +185,17 @@ class CertificateId:
                 break
         return new_cert_id
 
+    def _canonical_tr(self):
+        new_cert_id = self.clean
+        for rule in rules["cc_cert_id"]["TR"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                prefix = groups["prefix"]
+                number = groups["number"]
+                new_cert_id = f"{prefix}/TSE-CCCS-{number}"
+                break
+        return new_cert_id
+
     def _canonical_no(self):
         new_cert_id = self.clean
         cert_num = int(new_cert_id.split("-")[1])
@@ -241,6 +252,7 @@ class CertificateId:
             "NL": self._canonical_nl,
             "AU": self._canonical_au,
             "KR": self._canonical_kr,
+            "TR": self._canonical_tr,
             # SG is canonical by default
             # IT is canonical by default
         }
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index ca32fe91..5204b89e 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -63,7 +63,7 @@ def test_canonicalize_jp():
     assert canonicalize("JISEC-CC-CRP-C0689-01-2020", "JP") == "JISEC-CC-CRP-C0689-01-2020"
 
 
-def test_canonicalize_KR():
+def test_canonicalize_kr():
     assert canonicalize("KECS-ISIS-0579-2015", "KR") == "KECS-ISIS-0579-2015"
     assert canonicalize("KECS-CISS-10-2023", "KR") == "KECS-CISS-0010-2023"
 
@@ -72,6 +72,11 @@ def test_canonicalize_no():
     assert canonicalize("SERTIT-12", "NO") == "SERTIT-012"
 
 
+def test_canonicalize_tr():
+    assert canonicalize("21.0.03.0.00.00/TSE-CCCS-85", "TR") == "21.0.03.0.00.00/TSE-CCCS-85"
+    assert canonicalize("21.0.03/TSE-CCCS-33", "TR") == "21.0.03/TSE-CCCS-33"
+
+
 def test_canonicalize_nl():
     assert canonicalize("NSCIB-CC-22-0428888-CR2", "NL") == "NSCIB-CC-22-0428888-CR2"
     assert canonicalize("NSCIB-CC-22-0428888", "NL") == "NSCIB-CC-22-0428888-CR"
-- 
cgit v1.3.1


From d1b16b7086c63ee9b837f9a5b77971d9d7db9874 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 15:47:21 +0100
Subject: Improve Canadian rules.

---
 src/sec_certs/rules.yaml                  | 12 ++++++++----
 src/sec_certs/sample/cc_certificate_id.py | 22 +++++++++++++++++-----
 tests/cc/test_cc_misc.py                  |  1 +
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 3cb5d4d3..13d81c18 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -41,10 +41,14 @@ cc_cert_id:
     # CCEVS-VR-10880-2018
     # CCEVS-VR-04-0082
   CA:
-    # '[0-9][0-9\-]+?-CR', # Canada
-    - "[0-9][0-9][0-9]-[347]-[0-9][0-9][0-9]?(?:-CR|P)?"  # Canada xxx-{347}-xxx (383-4-438, 383-4-82-CR, 383-4-422P)
-    - "[0-9][0-9][0-9][ -](?:EWA|LSS|CCS)(?:[ -]20[0-9][0-9])?"  # Canada (522-EWA-2020, 524 LSS 2020, 503-LSS)
-    - "[0-9][0-9][0-9](?:%20|-)(?:EWA|LSS|CCS)(?:%20|-)(?:20[0-9][0-9]%20|)CR%20v[0-9]\\.[0-9]" # Canada filename with space (518-LSS%20CR%20v1.0)
+    - "(?P<number1>[0-9]+)[ -](?P<digit>[0-9])[ -](?P<number2>[0-9]+)(?:-CR|P)?"
+    - "(?P<number>[0-9]+)[ -](?P<lab>EWA|LSS|CCS)(?:[ -](?P<year>[0-9]+))?"
+    # Examples:
+    # 383-4-123-CR
+    # 383-4-123P
+    # 522 EWA 2020
+    # Filename rule:
+    #- "[0-9][0-9][0-9](?:%20|-)(?:EWA|LSS|CCS)(?:%20|-)(?:20[0-9][0-9]%20|)CR%20v[0-9]\\.[0-9]" # Canada filename with space (518-LSS%20CR%20v1.0)
   UK:
     - "CRP(?P<counter>[0-9]+[A-Z]?)"
     - "CERTIFICATION REPORT No. P(?P<counter>[0-9]+[A-Z]?)"
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 901629c7..e5f86bba 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -151,11 +151,23 @@ class CertificateId:
 
     def _canonical_ca(self):
         new_cert_id = self.clean
-        if new_cert_id.endswith("-CR"):
-            new_cert_id = new_cert_id[:-3]
-        if new_cert_id.endswith("P"):
-            new_cert_id = new_cert_id[:-1]
-        return new_cert_id.replace(" ", "-")
+        for rule in rules["cc_cert_id"]["CA"]:
+            if match := re.match(rule, new_cert_id):
+                groups = match.groupdict()
+                if "lab" in groups:
+                    year = _parse_year(groups.get("year"))
+                    number = groups["number"]
+                    lab = groups["lab"]
+                    new_cert_id = f"{number}-{lab}"
+                    if year:
+                        new_cert_id += f"-{year}"
+                else:
+                    number1 = groups["number1"]
+                    digit = groups["digit"]
+                    number2 = groups["number2"]
+                    new_cert_id = f"{number1}-{digit}-{number2}"
+                break
+        return new_cert_id
 
     def _canonical_jp(self):
         new_cert_id = self.clean
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index 5204b89e..5e67a58f 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -55,6 +55,7 @@ def test_canonicalize_au():
 def test_canonicalize_ca():
     assert canonicalize("383-4-123-CR", "CA") == "383-4-123"
     assert canonicalize("383-4-123P", "CA") == "383-4-123"
+    assert canonicalize("522 EWA 2020", "CA") == "522-EWA-2020"
 
 
 def test_canonicalize_jp():
-- 
cgit v1.3.1


From b05b86c32eb6068be317af1ce15e032f88d48807 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 15:55:36 +0100
Subject: Add certificateId meta property.

---
 src/sec_certs/sample/cc_certificate_id.py | 18 ++++++++++++++++++
 tests/cc/test_cc_misc.py                  |  8 +++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index e5f86bba..023e6e54 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import re
 from dataclasses import dataclass
+from functools import cached_property
 
 from sec_certs.cert_rules import rules
 
@@ -27,6 +28,13 @@ class CertificateId:
     scheme: str
     raw: str
 
+    @cached_property
+    def meta(self):
+        for rule in rules["cc_cert_id"][self.scheme]:
+            if match := re.match(rule, self.clean):
+                return match.groupdict()
+        return {}
+
     def _canonical_fr(self) -> str:
         new_cert_id = self.clean
         for rule in rules["cc_cert_id"]["FR"]:
@@ -274,6 +282,16 @@ class CertificateId:
         else:
             return self.clean
 
+    def __hash__(self):
+        return hash((self.scheme, self.raw))
+
+    def __eq__(self, other):
+        if isinstance(other, str):
+            return self.canonical == other
+        if not isinstance(other, CertificateId):
+            return False
+        return self.canonical == other.canonical and self.scheme == other.scheme
+
 
 def canonicalize(cert_id_str: str, scheme: str) -> str:
     return CertificateId(scheme, cert_id_str).canonical
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index 5e67a58f..5de22216 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -1,4 +1,10 @@
-from sec_certs.sample.cc_certificate_id import canonicalize
+from sec_certs.sample.cc_certificate_id import CertificateId, canonicalize
+
+
+def test_meta_parse():
+    i = CertificateId("FR", "Rapport de certification 2001/02v2")
+    assert "year" in i.meta
+    assert i.meta["year"] == "2001"
 
 
 def test_canonicalize_fr():
-- 
cgit v1.3.1


From 9ad30df4652a3246bb316a9d98cf96e102745768 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 17:38:50 +0100
Subject: Add filename cert id rules.

---
 src/sec_certs/rules.yaml                  | 37 +++++++++++++++++++++++++++++++
 src/sec_certs/sample/cc_certificate_id.py |  2 +-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 13d81c18..7cd5996e 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -121,6 +121,43 @@ cc_cert_id:
     # Certificate Number: 37/2006    !mistake
     # Certification Report 97/76     !short year
 
+#####
+# Common Criteria certificate IDs as they appear in report filenames, grouped by scheme (Alpha-2 ISO country code).
+#####
+cc_filename_cert_id:
+  DE:
+    - "(?P<counter>[0-9]{3,5})(?:(?P<version>[vV][0-9]))?a?(?:_pdf)?"
+    - "(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2})_(?P<counter>[0-9]{3,5})(?:(?P<version>[vV][0-9]))?a?(?:_pdf)?"
+  FR:
+    - "(?P<year>[0-9]{4})[_-](?P<counter>[0-9]{2})([vV](?P<version>[0-9]))?"
+    - "(?P<year>[0-9]{2})(?P<counter>[0-9]{2})([vV](?P<version>[0-9]))?"
+  NL:
+    - "(?:NSCIB-|CC-|NSCIB-CC-)(?P<core>((?P<year>[0-9]{2})-)?(?:-?[0-9]+)+)(?:-?(?P<doc>(?:CR|MA|MR)[0-9]*))?"
+  "NO":
+    - "SERTIT-(?P<counter>[0-9]+)"
+  US:
+  CA:
+    - "(?P<number1>[0-9]+)[ -](?P<digit>[0-9])[ -](?P<number2>[0-9]+)(?:-CR|P)?"
+    - "(?P<number>[0-9]+)[ -](?P<lab>EWA|LSS|CCS)(?:[ -](?P<year>[0-9]+))?"
+  UK:
+    - "CRP(?P<counter>[0-9]+[A-Z]?)"
+  ES:
+    - "(?P<year>[0-9]{4})[-‐](?P<project>[0-9]+)[-‐]INF[-‐](?P<counter>[0-9]+)[ -‐_]{1,2}[vV](?P<version>[0-9])"
+  KR:
+    - "(?P<word>ISIS|NISS|CISS)[-‐](?P<counter>[0-9]{2,4})(?:[-‐](?P<year>[0-9]{4}))?"
+  JP:
+    - "[cC](?P<counter>[0-9]+)"
+  MY:
+    - "ISCB-(?P<digit>[0-9])-RPT-C(?P<counter>[0-9]{3})-CR(?:-[0-9])?-(?P<version>[vV][0-9][a-z]?)"
+  IT:
+  TR:
+  SE:
+    - "CR(?P<year>[0-9]{4})(?P<counter>[0-9]{2,3})"
+  IN:
+  SG:
+  AU:
+    - "(?P<year>[0-9]{2,4})_(?P<counter>[0-9]+)"
+
 #####
 # Common Criteria protection profile IDs, grouped by certification body (e.g. BSI)
 #####
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 023e6e54..584039b6 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -19,7 +19,7 @@ def _parse_year(year: str | None) -> int | None:
         return y
 
 
-@dataclass(eq=True, frozen=True)
+@dataclass(frozen=True)
 class CertificateId:
     """
     A Common Criteria certificate id.
-- 
cgit v1.3.1


From 836139707bc2a826a1ff1c900c6e92854cb3d8bb Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 18:07:10 +0100
Subject: Move the meta -> str functions out of cert_id class.

---
 src/sec_certs/rules.yaml                  |   2 +-
 src/sec_certs/sample/cc_certificate_id.py | 414 ++++++++++++++----------------
 tests/test_common.py                      |   2 +-
 3 files changed, 188 insertions(+), 230 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 7cd5996e..a0e1c82d 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -5,7 +5,7 @@
 #####
 cc_cert_id:
   DE:
-    - "BSI-DSZ-CC-(?:(?P<s>S)-)?(?P<counter>[0-9]{3,5})-(?:(?P<version>[vV][0-9])-)?(?P<year>[0-9]{4})?(?:-(?P<doc>(?:RA|MA)(?:-[0-9]+)?))?"
+    - "BSI-DSZ-CC-(?:(?P<s>S)-)?(?P<counter>[0-9]{3,5})-?(?:(?P<version>[vV][0-9])-)?(?P<year>[0-9]{4})?(?:-(?P<doc>(?:RA|MA)(?:-[0-9]+)?))?"
     # Examples:
     # BSI-DSZ-CC-1004
     # BSI-DSZ-CC-0973-2016
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 584039b6..ee715082 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -19,6 +19,169 @@ def _parse_year(year: str | None) -> int | None:
         return y
 
 
+def FR(meta) -> str:
+    year = _parse_year(meta["year"])
+    counter = meta["counter"]
+    doc = meta.get("doc")
+    version = meta.get("version")
+    cert_id = f"ANSSI-CC-{year}/{counter}"
+    if doc:
+        cert_id += f"-{doc}"
+    if version:
+        cert_id += f"v{version}"
+    return cert_id
+
+
+def DE(meta) -> str:
+    s = meta.get("s")
+    counter = meta["counter"]
+    version = meta.get("version")
+    year = _parse_year(meta.get("year"))
+    doc = meta.get("doc")
+    cert_id = "BSI-DSZ-CC"
+    if s:
+        cert_id += f"-{s}"
+    cert_id += f"-{counter}"
+    if version:
+        cert_id += f"-{version.upper()}"
+    if year:
+        cert_id += f"-{year}"
+    if doc:
+        cert_id += f"-{doc}"
+    return cert_id
+
+
+def US(meta) -> str:
+    year = _parse_year(meta["year"])
+    counter = meta["counter"]
+    cc = meta.get("cc")
+    vid = meta.get("VID")
+    cert_id = "CCEVS-VR"
+    if cc:
+        cert_id += f"-{cc}"
+    if vid:
+        cert_id += f"-{vid}"
+    cert_id += f"-{counter}"
+    cert_id += f"-{year}"
+    return cert_id
+
+
+def MY(meta) -> str:
+    digit = meta["digit"]
+    counter = meta["counter"]
+    version = meta["version"]
+    return f"ISCB-{digit}-RPT-C{counter}-CR-{version.lower()}"
+
+
+def ES(meta) -> str:
+    year = _parse_year(meta["year"])
+    project = meta["project"]
+    counter = meta["counter"]
+    # Version is intentionally cut here, as it seems to refer to an internal version of the report.
+    # version = groups["version"]
+    return f"{year}-{project}-INF-{counter}"
+
+
+def IN(meta) -> str:
+    lab = meta["lab"]
+    vendor = meta["vendor"]
+    level = meta["level"]
+    number1, number2 = meta["number1"], meta["number2"]
+    return f"IC3S/{lab}/{vendor}/{level}/{number1}/{number2}"
+
+
+def SE(meta) -> str:
+    year = _parse_year(meta["year"])
+    counter = int(meta["counter"])
+    return f"CSEC{year}{counter:03}"
+
+
+def UK(meta) -> str:
+    counter = meta["counter"]
+    return f"CRP{counter}"
+
+
+def CA(meta) -> str:
+    if "lab" in meta:
+        year = _parse_year(meta.get("year"))
+        number = meta["number"]
+        lab = meta["lab"]
+        cert_id = f"{number}-{lab}"
+        if year:
+            cert_id += f"-{year}"
+        return cert_id
+    else:
+        number1 = meta["number1"]
+        digit = meta["digit"]
+        number2 = meta["number2"]
+        return f"{number1}-{digit}-{number2}"
+
+
+def JP(meta) -> str:
+    counter = meta["counter"]
+    digit = meta.get("digit")
+    year = _parse_year(meta.get("year"))
+    cert_id = f"JISEC-CC-CRP-C{counter}"
+    if digit:
+        cert_id += f"-{digit}"
+    if year:
+        cert_id += f"-{year}"
+    return cert_id
+
+
+def KR(meta) -> str:
+    word = meta["word"]
+    counter = int(meta["counter"])
+    year = _parse_year(meta["year"])
+    return f"KECS-{word}-{counter:04}-{year}"
+
+
+def TR(meta) -> str:
+    prefix = meta["prefix"]
+    number = meta["number"]
+    return f"{prefix}/TSE-CCCS-{number}"
+
+
+def NO(meta) -> str:
+    counter = int(meta["counter"])
+    return f"SERTIT-{counter:03}"
+
+
+def NL(meta) -> str:
+    core = meta["core"]
+    doc = meta.get("doc")
+    if doc is None:
+        doc = "CR"
+    return f"NSCIB-CC-{core}-{doc}"
+
+
+def AU(meta) -> str:
+    counter = meta["counter"]
+    year_s = meta["year"]
+    if len(year_s) < len(counter):
+        # Hack for some mistakes in their ordering
+        year_s, counter = counter, year_s
+    year = _parse_year(year_s)
+    return f"Certificate Number: {year}/{counter}"
+
+
+def SG(meta) -> str:
+    year = meta["year"]
+    counter = meta["counter"]
+    return f"CSA_CC_{year}{counter}"
+
+
+def IT(meta) -> str:
+    lab = meta.get("lab")
+    counter = meta["counter"]
+    year = _parse_year(meta["year"])
+    cert_id = "OCSI/CERT/"
+    if lab:
+        cert_id += f"{lab}/"
+    cert_id += f"{counter}/{year}/RC"
+    return cert_id
+
+
 @dataclass(frozen=True)
 class CertificateId:
     """
@@ -35,215 +198,6 @@ class CertificateId:
                 return match.groupdict()
         return {}
 
-    def _canonical_fr(self) -> str:
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["FR"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                year = _parse_year(groups["year"])
-                counter = groups["counter"]
-                doc = groups.get("doc")
-                version = groups.get("version")
-                new_cert_id = f"ANSSI-CC-{year}/{counter}"
-                if doc:
-                    new_cert_id += f"-{doc}"
-                if version:
-                    new_cert_id += f"v{version}"
-                break
-
-        return new_cert_id
-
-    def _canonical_de(self) -> str:
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["DE"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                s = groups.get("s")
-                counter = groups["counter"]
-                version = groups.get("version")
-                year = _parse_year(groups.get("year"))
-                doc = groups.get("doc")
-                new_cert_id = "BSI-DSZ-CC"
-                if s:
-                    new_cert_id += f"-{s}"
-                new_cert_id += f"-{counter}"
-                if version:
-                    new_cert_id += f"-{version.upper()}"
-                if year:
-                    new_cert_id += f"-{year}"
-                if doc:
-                    new_cert_id += f"-{doc}"
-                break
-
-        return new_cert_id
-
-    def _canonical_us(self) -> str:
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["US"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                year = _parse_year(groups["year"])
-                counter = groups["counter"]
-                cc = groups.get("cc")
-                vid = groups.get("VID")
-                new_cert_id = "CCEVS-VR"
-                if cc:
-                    new_cert_id += f"-{cc}"
-                if vid:
-                    new_cert_id += f"-{vid}"
-                new_cert_id += f"-{counter}"
-                new_cert_id += f"-{year}"
-                break
-
-        return new_cert_id
-
-    def _canonical_my(self) -> str:
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["MY"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                digit = groups["digit"]
-                counter = groups["counter"]
-                version = groups["version"]
-                new_cert_id = f"ISCB-{digit}-RPT-C{counter}-CR-{version.lower()}"
-                break
-
-        return new_cert_id
-
-    def _canonical_es(self) -> str:
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["ES"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                year = _parse_year(groups["year"])
-                project = groups["project"]
-                counter = groups["counter"]
-                # Version is intentionally cut here, as it seems to refer to an internal version of the report.
-                # version = groups["version"]
-                new_cert_id = f"{year}-{project}-INF-{counter}"
-                break
-        return new_cert_id
-
-    def _canonical_in(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["IN"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                lab = groups["lab"]
-                vendor = groups["vendor"]
-                level = groups["level"]
-                number1, number2 = groups["number1"], groups["number2"]
-                new_cert_id = f"IC3S/{lab}/{vendor}/{level}/{number1}/{number2}"
-                break
-        return new_cert_id
-
-    def _canonical_se(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["SE"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                year = _parse_year(groups["year"])
-                counter = int(groups["counter"])
-                new_cert_id = f"CSEC{year}{counter:03}"
-        return new_cert_id
-
-    def _canonical_uk(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["UK"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                counter = groups["counter"]
-                new_cert_id = f"CRP{counter}"
-                break
-        return new_cert_id
-
-    def _canonical_ca(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["CA"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                if "lab" in groups:
-                    year = _parse_year(groups.get("year"))
-                    number = groups["number"]
-                    lab = groups["lab"]
-                    new_cert_id = f"{number}-{lab}"
-                    if year:
-                        new_cert_id += f"-{year}"
-                else:
-                    number1 = groups["number1"]
-                    digit = groups["digit"]
-                    number2 = groups["number2"]
-                    new_cert_id = f"{number1}-{digit}-{number2}"
-                break
-        return new_cert_id
-
-    def _canonical_jp(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["JP"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                counter = groups["counter"]
-                digit = groups.get("digit")
-                year = _parse_year(groups.get("year"))
-                new_cert_id = f"JISEC-CC-CRP-C{counter}"
-                if digit:
-                    new_cert_id += f"-{digit}"
-                if year:
-                    new_cert_id += f"-{year}"
-                break
-        return new_cert_id
-
-    def _canonical_kr(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["KR"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                word = groups["word"]
-                counter = int(groups["counter"])
-                year = _parse_year(groups["year"])
-                new_cert_id = f"KECS-{word}-{counter:04}-{year}"
-                break
-        return new_cert_id
-
-    def _canonical_tr(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["TR"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                prefix = groups["prefix"]
-                number = groups["number"]
-                new_cert_id = f"{prefix}/TSE-CCCS-{number}"
-                break
-        return new_cert_id
-
-    def _canonical_no(self):
-        new_cert_id = self.clean
-        cert_num = int(new_cert_id.split("-")[1])
-        return f"SERTIT-{cert_num:03}"
-
-    def _canonical_nl(self):
-        new_cert_id = self.clean
-        if new_cert_id.startswith("CC-"):
-            new_cert_id = f"NSCIB-{new_cert_id}"
-        if not re.match(".*-(CR|MA|MR)[0-9]*$", new_cert_id):
-            new_cert_id = f"{new_cert_id}-CR"
-        return new_cert_id
-
-    def _canonical_au(self):
-        new_cert_id = self.clean
-        for rule in rules["cc_cert_id"]["AU"]:
-            if match := re.match(rule, new_cert_id):
-                groups = match.groupdict()
-                counter = groups["counter"]
-                year_s = groups["year"]
-                if len(year_s) < len(counter):
-                    # Hack for some mistakes in their ordering
-                    year_s, counter = counter, year_s
-                year = _parse_year(year_s)
-                new_cert_id = f"Certificate Number: {year}/{counter}"
-                break
-        return new_cert_id
-
     @property
     def clean(self) -> str:
         """
@@ -258,29 +212,33 @@ class CertificateId:
         """
         # We have rules for some schemes to make canonical cert_ids.
         schemes = {
-            "FR": self._canonical_fr,
-            "DE": self._canonical_de,
-            "US": self._canonical_us,
-            "MY": self._canonical_my,
-            "ES": self._canonical_es,
-            "IN": self._canonical_in,
-            "SE": self._canonical_se,
-            "UK": self._canonical_uk,
-            "CA": self._canonical_ca,
-            "JP": self._canonical_jp,
-            "NO": self._canonical_no,
-            "NL": self._canonical_nl,
-            "AU": self._canonical_au,
-            "KR": self._canonical_kr,
-            "TR": self._canonical_tr,
-            # SG is canonical by default
-            # IT is canonical by default
+            "FR": FR,
+            "DE": DE,
+            "US": US,
+            "MY": MY,
+            "ES": ES,
+            "IN": IN,
+            "SE": SE,
+            "UK": UK,
+            "CA": CA,
+            "JP": JP,
+            "NO": NO,
+            "NL": NL,
+            "AU": AU,
+            "KR": KR,
+            "TR": TR,
+            "SG": SG,
+            "IT": IT,
         }
+        clean = self.clean
 
         if self.scheme in schemes:
-            return schemes[self.scheme]()
+            return schemes[self.scheme](self.meta)
         else:
-            return self.clean
+            return clean
+
+    def __str__(self):
+        return self.canonical
 
     def __hash__(self):
         return hash((self.scheme, self.raw))
diff --git a/tests/test_common.py b/tests/test_common.py
index fa7a3775..7b29dd8d 100644
--- a/tests/test_common.py
+++ b/tests/test_common.py
@@ -5,5 +5,5 @@ def test_rules():
     assert "cc_cert_id" in cc_rules
     assert "fips_cert_id" in fips_rules
     for rule_group in rules:
-        if rule_group not in ("cc_rules", "fips_rules"):
+        if rule_group not in ("cc_rules", "fips_rules", "cc_filename_cert_id"):
             assert rule_group in cc_rules or rule_group in fips_rules
-- 
cgit v1.3.1


From 4e24bb278c9da4e4f6c75f92187c6173d4269010 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 18:12:34 +0100
Subject: Extract and reconstruct cert ids from filenames.

Before this, the regular cert_id regexes were used to extract
the cert_id from the report filename. However, the filenames often
do not use the same cert_id format, but contain all of the information
necessary to reconstruct the cert_id, but with different order for example.

This commit along with those before it introduce a new set of regular
expressions that better match the ones in the filenames. To extract
the correctly formatted canonical cert_id, the regexes are used to
obtain the parts of the cert_id (using named groups in regexes) and
those are then reconstructed into a canonical version of the cert_id
via one of the scheme-dependent functions.
---
 src/sec_certs/sample/cc.py                | 14 +++++++----
 src/sec_certs/sample/cc_certificate_id.py | 42 ++++++++++++++++---------------
 2 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 6ee2bd1b..8948bb76 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -19,7 +19,7 @@ import sec_certs.utils.pdf
 import sec_certs.utils.sanitization
 from sec_certs import constants
 from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, cc_rules, rules, security_level_csv_scan
-from sec_certs.sample.cc_certificate_id import canonicalize
+from sec_certs.sample.cc_certificate_id import canonicalize, schemes
 from sec_certs.sample.certificate import Certificate, References, logger
 from sec_certs.sample.certificate import Heuristics as BaseHeuristics
 from sec_certs.sample.certificate import PdfData as BasePdfData
@@ -337,13 +337,17 @@ class CCCertificate(
             """
             if not self.report_filename:
                 return {}
-            scheme_rules = rules["cc_cert_id"][scheme]
+            scheme_filename_rules = rules["cc_filename_cert_id"][scheme]
+            scheme_meta = schemes[scheme]
             matches: Counter = Counter()
-            for rule in scheme_rules:
+            for rule in scheme_filename_rules:
                 match = re.search(rule, self.report_filename)
                 if match:
-                    cert_id = normalize_match_string(match.group())
-                    matches[cert_id] += 1
+                    try:
+                        cert_id = scheme_meta(match.groupdict())
+                        matches[cert_id] += 1
+                    except Exception:
+                        continue
             if not matches:
                 return {}
             total = max(matches.values())
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index ee715082..1fb7b046 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -182,6 +182,28 @@ def IT(meta) -> str:
     return cert_id
 
 
+# We have rules for some schemes to make canonical cert_ids.
+schemes = {
+    "FR": FR,
+    "DE": DE,
+    "US": US,
+    "MY": MY,
+    "ES": ES,
+    "IN": IN,
+    "SE": SE,
+    "UK": UK,
+    "CA": CA,
+    "JP": JP,
+    "NO": NO,
+    "NL": NL,
+    "AU": AU,
+    "KR": KR,
+    "TR": TR,
+    "SG": SG,
+    "IT": IT,
+}
+
+
 @dataclass(frozen=True)
 class CertificateId:
     """
@@ -210,26 +232,6 @@ class CertificateId:
         """
         The canonical version of this certificate id.
         """
-        # We have rules for some schemes to make canonical cert_ids.
-        schemes = {
-            "FR": FR,
-            "DE": DE,
-            "US": US,
-            "MY": MY,
-            "ES": ES,
-            "IN": IN,
-            "SE": SE,
-            "UK": UK,
-            "CA": CA,
-            "JP": JP,
-            "NO": NO,
-            "NL": NL,
-            "AU": AU,
-            "KR": KR,
-            "TR": TR,
-            "SG": SG,
-            "IT": IT,
-        }
         clean = self.clean
 
         if self.scheme in schemes:
-- 
cgit v1.3.1


From 40d89ff77f3e59455082da48e49b0e9f7b57d78a Mon Sep 17 00:00:00 2001
From: J08nY
Date: Mon, 5 Feb 2024 18:28:01 +0100
Subject: Fix broken filename rules.

---
 src/sec_certs/rules.yaml   | 4 ++--
 src/sec_certs/sample/cc.py | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index a0e1c82d..4305b8cd 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -126,11 +126,11 @@ cc_cert_id:
 #####
 cc_filename_cert_id:
   DE:
-    - "(?P<counter>[0-9]{3,5})(?:(?P<version>[vV][0-9]))?a?(?:_pdf)?"
+    #- "(?P<counter>[0-9]{3,5})(?:(?P<version>[vV][0-9]))?a?(?:_pdf)?"
     - "(?P<year>[0-9]{4})(?P<month>[0-9]{2})(?P<day>[0-9]{2})_(?P<counter>[0-9]{3,5})(?:(?P<version>[vV][0-9]))?a?(?:_pdf)?"
   FR:
     - "(?P<year>[0-9]{4})[_-](?P<counter>[0-9]{2})([vV](?P<version>[0-9]))?"
-    - "(?P<year>[0-9]{2})(?P<counter>[0-9]{2})([vV](?P<version>[0-9]))?"
+    #- "(?P<year>[0-9]{2})(?P<counter>[0-9]{2})([vV](?P<version>[0-9]))?"
   NL:
     - "(?:NSCIB-|CC-|NSCIB-CC-)(?P<core>((?P<year>[0-9]{2})-)?(?:-?[0-9]+)+)(?:-?(?P<doc>(?:CR|MA|MR)[0-9]*))?"
   "NO":
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 8948bb76..ef1ff984 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -338,13 +338,16 @@ class CCCertificate(
             if not self.report_filename:
                 return {}
             scheme_filename_rules = rules["cc_filename_cert_id"][scheme]
+            if not scheme_filename_rules:
+                return {}
             scheme_meta = schemes[scheme]
             matches: Counter = Counter()
             for rule in scheme_filename_rules:
                 match = re.search(rule, self.report_filename)
                 if match:
                     try:
-                        cert_id = scheme_meta(match.groupdict())
+                        meta = match.groupdict()
+                        cert_id = scheme_meta(meta)
                         matches[cert_id] += 1
                     except Exception:
                         continue
-- 
cgit v1.3.1


From 8d989b6de8465d3cf8af6502ab9cfb6ee8cbdded Mon Sep 17 00:00:00 2001
From: dependabot[bot]
Date: Tue, 6 Feb 2024 03:03:01 +0000
Subject: chore(deps): bump cryptography from 41.0.6 to 42.0.0 in /requirements

Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.6 to 42.0.0.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/41.0.6...42.0.0)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>---
 requirements/all_requirements.txt  | 2 +-
 requirements/dev_requirements.txt  | 2 +-
 requirements/nlp_requirements.txt  | 2 +-
 requirements/requirements.txt      | 2 +-
 requirements/test_requirements.txt | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index d96e2811..ae9e742c 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -94,7 +94,7 @@ coverage[toml]==7.3.2
     # via
     #   pytest-cov
     #   sec-certs (./../pyproject.toml)
-cryptography==41.0.6
+cryptography==42.0.0
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index 75d23714..f340bbd9 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -71,7 +71,7 @@ coverage[toml]==7.3.2
     # via
     #   coverage
     #   pytest-cov
-cryptography==41.0.6
+cryptography==42.0.0
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
index bc558064..f06f83c8 100644
--- a/requirements/nlp_requirements.txt
+++ b/requirements/nlp_requirements.txt
@@ -73,7 +73,7 @@ contourpy==1.2.0
     # via
     #   bokeh
     #   matplotlib
-cryptography==41.0.6
+cryptography==42.0.0
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 93645663..3ecf9f39 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -41,7 +41,7 @@ confection==0.1.3
     #   weasel
 contourpy==1.2.0
     # via matplotlib
-cryptography==41.0.6
+cryptography==42.0.0
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 1922b83b..3a2a5198 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -45,7 +45,7 @@ coverage[toml]==7.3.2
     # via
     #   pytest-cov
     #   sec-certs (./../pyproject.toml)
-cryptography==41.0.6
+cryptography==42.0.0
     # via pypdf
 cycler==0.12.1
     # via matplotlib
-- 
cgit v1.3.1


From 90a8cf99f96d9174b5bbfaa3a881b4792ecb3453 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Tue, 6 Feb 2024 15:56:24 +0100
Subject: Add tests fot IT and SG canonicalization.

---
 tests/cc/test_cc_misc.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index 5de22216..c06b26ed 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -36,10 +36,18 @@ def test_canonicalize_es():
     assert canonicalize("2011-14-INF-1095-v1", "ES") == "2011-14-INF-1095"
 
 
+def test_canonicalize_sg():
+    assert canonicalize("CSA_CC_21005", "SG") == "CSA_CC_21005"
+
+
 def test_canonicalize_in():
     assert canonicalize("IC3S/KOL01/ADVA/EAL2/0520/0021 /CR", "IN") == "IC3S/KOL01/ADVA/EAL2/0520/0021"
 
 
+def test_canonicalize_it():
+    assert canonicalize("OCSI/CERT/TEC/02/2009/RC", "IT") == "OCSI/CERT/TEC/02/2009/RC"
+
+
 def test_canonicalize_se():
     assert canonicalize("CSEC2017020", "SE") == "CSEC2017020"
     assert canonicalize("CSEC 2017020", "SE") == "CSEC2017020"
-- 
cgit v1.3.1


From ec0162c8f82b00e29c1551109b3a3a61d37adbac Mon Sep 17 00:00:00 2001
From: J08nY
Date: Tue, 6 Feb 2024 16:01:49 +0100
Subject: Add more tests for cert_ids.

---
 tests/cc/test_cc_misc.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index c06b26ed..bf880f95 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -5,6 +5,8 @@ def test_meta_parse():
     i = CertificateId("FR", "Rapport de certification 2001/02v2")
     assert "year" in i.meta
     assert i.meta["year"] == "2001"
+    assert i.meta["counter"] == "02"
+    assert i.meta["version"] == "2"
 
 
 def test_canonicalize_fr():
@@ -96,3 +98,11 @@ def test_canonicalize_nl():
     assert canonicalize("NSCIB-CC-22-0428888-CR2", "NL") == "NSCIB-CC-22-0428888-CR2"
     assert canonicalize("NSCIB-CC-22-0428888", "NL") == "NSCIB-CC-22-0428888-CR"
     assert canonicalize("CC-22-0428888", "NL") == "NSCIB-CC-22-0428888-CR"
+
+
+def test_certid_compare():
+    cid1 = CertificateId("AU", "Certification Report 2007/02")
+    cid2 = CertificateId("AU", "Certificate Number: 02/2007")
+    cid3 = CertificateId("AU", "Certificate Number: 05/2007")
+    assert cid1 == cid2
+    assert cid1 != cid3
-- 
cgit v1.3.1


From d1911fce8b9aa4006659d55740c22d3e8a40dbb5 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Wed, 7 Feb 2024 10:09:54 +0100
Subject: Fix cert_id canonicalization.

---
 src/sec_certs/rules.yaml   |  4 ++--
 src/sec_certs/sample/cc.py | 22 ++++++++++++++++++----
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 4305b8cd..250c45b9 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -70,7 +70,7 @@ cc_cert_id:
     # KECS-CISS-1210-2023
   JP:
     - "(?:CRP|ACR)-C(?P<counter>[0-9]+)-(?P<digit>[0-9]+)"
-    - "JISEC-CC-CRP-C(?P<counter>[0-9]+)-(?P<digit>[0-9]+)-(?P<year>[0-9]{4})"
+    - "JISEC-CC-CRP-C(?P<counter>[0-9]+)(?:-(?P<digit>[0-9]{2}))?(?:-(?P<year>[0-9]{4}))?"
     - "Certification No. [cC](?P<counter>[0-9]+)"
     # Examples:
     # CRP-C0595-01
@@ -144,7 +144,7 @@ cc_filename_cert_id:
   ES:
     - "(?P<year>[0-9]{4})[-‐](?P<project>[0-9]+)[-‐]INF[-‐](?P<counter>[0-9]+)[ -‐_]{1,2}[vV](?P<version>[0-9])"
   KR:
-    - "(?P<word>ISIS|NISS|CISS)[-‐](?P<counter>[0-9]{2,4})(?:[-‐](?P<year>[0-9]{4}))?"
+    - "(?P<word>ISIS|NISS|CISS)[-‐](?P<counter>[0-9]{2,4})[-‐](?P<year>[0-9]{4})"
   JP:
     - "[cC](?P<counter>[0-9]+)"
   MY:
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index ef1ff984..904c794d 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -417,14 +417,28 @@ class CCCertificate(
             candidates: dict[str, float] = defaultdict(lambda: 0.0)
             # TODO: Add heuristic based on ordering of ids (and extracted year + increment)
             # TODO: Add heuristic based on length
+            # TODO: Add heuristic based on id "richness", we want to prefer IDs that have more components.
+            # If we cannot canonicalize, just skip that ID.
             for candidate, count in frontpage_id.items():
-                candidates[canonicalize(candidate, scheme)] += count * 1.5
+                try:
+                    candidates[canonicalize(candidate, scheme)] += count * 1.5
+                except Exception:
+                    continue
             for candidate, count in metadata_id.items():
-                candidates[canonicalize(candidate, scheme)] += count * 1.2
+                try:
+                    candidates[canonicalize(candidate, scheme)] += count * 1.2
+                except Exception:
+                    continue
             for candidate, count in keywords_id.items():
-                candidates[canonicalize(candidate, scheme)] += count * 1.0
+                try:
+                    candidates[canonicalize(candidate, scheme)] += count * 1.0
+                except Exception:
+                    continue
             for candidate, count in filename_id.items():
-                candidates[canonicalize(candidate, scheme)] += count * 1.0
+                try:
+                    candidates[canonicalize(candidate, scheme)] += count * 1.0
+                except Exception:
+                    continue
             return candidates
 
     @dataclass
-- 
cgit v1.3.1


From 131489e8a1f32d5119fbb2a1283827fe6f045c74 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Wed, 7 Feb 2024 21:31:00 +0100
Subject: Handle some strict and loose rules after cleanup.

---
 src/sec_certs/rules.yaml                  | 5 +++--
 src/sec_certs/sample/cc_certificate_id.py | 5 +++--
 tests/cc/test_cc_misc.py                  | 1 +
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/sec_certs/rules.yaml b/src/sec_certs/rules.yaml
index 250c45b9..43a309e1 100644
--- a/src/sec_certs/rules.yaml
+++ b/src/sec_certs/rules.yaml
@@ -35,13 +35,14 @@ cc_cert_id:
     # SERTIT-101
   US:
     - "CCEVS-VR-(?:(?P<cc>CC)-)?(?:(?P<VID>VID)-?)?(?P<year>[0-9]{2})-(?P<counter>[0-9]+)"
-    - "CCEVS-VR-(?:(?P<cc>CC)-)?(?:(?P<VID>VID)-?)?(?P<counter>[0-9]{4,5})-(?P<year>[0-9]{4})?"
+    - "CCEVS-VR-(?:(?P<cc>CC)-)?(?:(?P<VID>VID)-?)?(?P<counter>[0-9]{4,5})(?:-(?P<year>[0-9]{4}))?"
     # Examples:
     # CCEVS-VR-VID10015-2008
     # CCEVS-VR-10880-2018
     # CCEVS-VR-04-0082
+    # CCEVS-VR-VID10318
   CA:
-    - "(?P<number1>[0-9]+)[ -](?P<digit>[0-9])[ -](?P<number2>[0-9]+)(?:-CR|P)?"
+    - "(?P<number1>383)[ -](?P<digit>[0-9])[ -](?P<number2>[0-9]+)(?:-CR|P)?"
     - "(?P<number>[0-9]+)[ -](?P<lab>EWA|LSS|CCS)(?:[ -](?P<year>[0-9]+))?"
     # Examples:
     # 383-4-123-CR
diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py
index 1fb7b046..c2b6daa2 100644
--- a/src/sec_certs/sample/cc_certificate_id.py
+++ b/src/sec_certs/sample/cc_certificate_id.py
@@ -52,17 +52,18 @@ def DE(meta) -> str:
 
 
 def US(meta) -> str:
-    year = _parse_year(meta["year"])
     counter = meta["counter"]
     cc = meta.get("cc")
     vid = meta.get("VID")
+    year = _parse_year(meta.get("year"))
     cert_id = "CCEVS-VR"
     if cc:
         cert_id += f"-{cc}"
     if vid:
         cert_id += f"-{vid}"
     cert_id += f"-{counter}"
-    cert_id += f"-{year}"
+    if year:
+        cert_id += f"-{year}"
     return cert_id
 
 
diff --git a/tests/cc/test_cc_misc.py b/tests/cc/test_cc_misc.py
index bf880f95..d67715f8 100644
--- a/tests/cc/test_cc_misc.py
+++ b/tests/cc/test_cc_misc.py
@@ -23,6 +23,7 @@ def test_canonicalize_de():
 
 
 def test_canonicalize_us():
+    assert canonicalize("CCEVS-VR-VID10015", "US") == "CCEVS-VR-VID-10015"
     assert canonicalize("CCEVS-VR-VID10015-2008", "US") == "CCEVS-VR-VID-10015-2008"
     assert canonicalize("CCEVS-VR-10880-2018", "US") == "CCEVS-VR-10880-2018"
     assert canonicalize("CCEVS-VR-04-0082", "US") == "CCEVS-VR-0082-2004"
-- 
cgit v1.3.1


From 4243fd6937bfdcab2dbcaa80e26fb8a03c579b0a Mon Sep 17 00:00:00 2001
From: J08nY
Date: Thu, 8 Feb 2024 00:58:58 +0100
Subject: Add extraction of certificate data.

---
 src/sec_certs/dataset/cc.py                        | 104 +++++++++++--
 src/sec_certs/sample/cc.py                         | 164 ++++++++++++++++++---
 tests/cc/test_cc_certificate.py                    |   4 -
 tests/data/cc/certificate/fictional_cert.json      |  14 +-
 .../maintenances/maintenance_updates.json          |  14 +-
 tests/data/cc/dataset/toy_dataset.json             |  42 +++++-
 6 files changed, 295 insertions(+), 47 deletions(-)

diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index 398f2435..30a6038a 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -143,6 +143,27 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         """
         return self.targets_dir / "txt"
 
+    @property
+    def certificates_dir(self) -> Path:
+        """
+        Returns directory that holds files associated with the certificates
+        """
+        return self.certs_dir / "certificates"
+
+    @property
+    def certificates_pdf_dir(self) -> Path:
+        """
+        Returns directory that holds PDFs associated with certificates
+        """
+        return self.certificates_dir / "pdf"
+
+    @property
+    def certificates_txt_dir(self) -> Path:
+        """
+        Returns directory that holds TXTs associated with certificates
+        """
+        return self.certificates_dir / "txt"
+
     @property
     def pp_dataset_path(self) -> Path:
         """
@@ -242,7 +263,14 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             self.auxiliary_datasets.mu_dset.root_dir = self.mu_dataset_dir
 
         for cert in self:
-            cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir)
+            cert.set_local_paths(
+                self.reports_pdf_dir,
+                self.targets_pdf_dir,
+                self.certificates_pdf_dir,
+                self.reports_txt_dir,
+                self.targets_txt_dir,
+                self.certificates_txt_dir,
+            )
         # TODO: This forgets to set local paths for other auxiliary datasets
 
     def _merge_certs(self, certs: dict[str, CCCertificate], cert_source: str | None = None) -> None:
@@ -531,6 +559,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     def _download_all_artifacts_body(self, fresh: bool = True) -> None:
         self._download_reports(fresh)
         self._download_targets(fresh)
+        self._download_certs(fresh)
 
     @staged(logger, "Downloading PDFs of CC certification reports.")
     def _download_reports(self, fresh: bool = True) -> None:
@@ -551,7 +580,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Downloading PDFs of CC security targets.")
     def _download_targets(self, fresh: bool = True) -> None:
         self.targets_pdf_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh)]
+        certs_to_process = [x for x in self if x.state.st_is_ok_to_download(fresh)]
 
         if not fresh and certs_to_process:
             logger.info(
@@ -564,6 +593,22 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Downloading PDFs of CC security targets",
         )
 
+    @staged(logger, "Downloading PDFs of CC certificates.")
+    def _download_certs(self, fresh: bool = True) -> None:
+        self.certificates_pdf_dir.mkdir(parents=True, exist_ok=True)
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_download(fresh)]
+
+        if not fresh and certs_to_process:
+            logger.info(
+                f"Downloading {len(certs_to_process)} PDFs of CC certificates for which previous download failed.."
+            )
+
+        cert_processing.process_parallel(
+            CCCertificate.download_pdf_cert,
+            certs_to_process,
+            progress_bar_desc="Downloading PDFs of CC certificates",
+        )
+
     @staged(logger, "Converting PDFs of certification reports to txt.")
     def _convert_reports_to_txt(self, fresh: bool = True) -> None:
         self.reports_txt_dir.mkdir(parents=True, exist_ok=True)
@@ -598,9 +643,28 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Converting PDFs of security targets to txt",
         )
 
+    @staged(logger, "Converting PDFs of certificates to txt.")
+    def _convert_certs_to_txt(self, fresh: bool = True) -> None:
+        self.certificates_txt_dir.mkdir(parents=True, exist_ok=True)
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_convert(fresh)]
+
+        if fresh:
+            logger.info("Converting PDFs of certificates to txt.")
+        if not fresh and certs_to_process:
+            logger.info(
+                f"Converting {len(certs_to_process)} PDFs of certificates to txt for which previous conversion failed."
+            )
+
+        cert_processing.process_parallel(
+            CCCertificate.convert_cert_pdf,
+            certs_to_process,
+            progress_bar_desc="Converting PDFs of security targets to txt",
+        )
+
     def _convert_all_pdfs_body(self, fresh: bool = True) -> None:
         self._convert_reports_to_txt(fresh)
         self._convert_targets_to_txt(fresh)
+        self._convert_certs_to_txt(fresh)
 
     @staged(logger, "Extracting report metadata")
     def _extract_report_metadata(self) -> None:
@@ -624,6 +688,17 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
+    @staged(logger, "Extracting cert metadata")
+    def _extract_cert_metadata(self) -> None:
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_analyze()]
+        processed_certs = cert_processing.process_parallel(
+            CCCertificate.extract_cert_pdf_metadata,
+            certs_to_process,
+            use_threading=False,
+            progress_bar_desc="Extracting cert metadata",
+        )
+        self.update_with_certs(processed_certs)
+
     def _extract_pdf_metadata(self) -> None:
         self._extract_report_metadata()
         self._extract_target_metadata()
@@ -639,20 +714,9 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    @staged(logger, "Extracting target frontpages")
-    def _extract_target_frontpage(self) -> None:
-        certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
-        processed_certs = cert_processing.process_parallel(
-            CCCertificate.extract_st_pdf_frontpage,
-            certs_to_process,
-            use_threading=False,
-            progress_bar_desc="Extracting target frontpages",
-        )
-        self.update_with_certs(processed_certs)
-
     def _extract_pdf_frontpage(self) -> None:
         self._extract_report_frontpage()
-        self._extract_target_frontpage()
+        # We have no frontpage extraction for targets or certificates themselves, only for the reports.
 
     @staged(logger, "Extracting report keywords")
     def _extract_report_keywords(self) -> None:
@@ -676,9 +740,21 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
+    @staged(logger, "Extracting cert keywords")
+    def _extract_cert_keywords(self) -> None:
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_analyze()]
+        processed_certs = cert_processing.process_parallel(
+            CCCertificate.extract_cert_pdf_keywords,
+            certs_to_process,
+            use_threading=False,
+            progress_bar_desc="Extracting cert keywords",
+        )
+        self.update_with_certs(processed_certs)
+
     def _extract_pdf_keywords(self) -> None:
         self._extract_report_keywords()
         self._extract_target_keywords()
+        self._extract_cert_keywords()
 
     def extract_data(self) -> None:
         logger.info("Extracting various data from certification artifacts")
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 904c794d..a4426d01 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -104,51 +104,71 @@ class CCCertificate(
 
         st_download_ok: bool  # Whether target download went OK
         report_download_ok: bool  # Whether report download went OK
+        cert_download_ok: bool  # Whether certificate download went OK
         st_convert_garbage: bool  # Whether initial target conversion resulted in garbage
         report_convert_garbage: bool  # Whether initial report conversion resulted in garbage
+        cert_convert_garbage: bool  # Whether initial certificate conversion resulted in garbage
         st_convert_ok: bool  # Whether overall target conversion went OK (either pdftotext or via OCR)
         report_convert_ok: bool  # Whether overall report conversion went OK (either pdftotext or via OCR)
+        cert_convert_ok: bool  # Whether overall certificate conversion went OK (either pdftotext or via OCR)
         st_extract_ok: bool  # Whether target extraction went OK
         report_extract_ok: bool  # Whether report extraction went OK
+        cert_extract_ok: bool  # Whether certificate extraction went OK
 
         st_pdf_hash: str | None
         report_pdf_hash: str | None
+        cert_pdf_hash: str | None
         st_txt_hash: str | None
         report_txt_hash: str | None
+        cert_txt_hash: str | None
 
         _st_pdf_path: Path | None = None
         _report_pdf_path: Path | None = None
+        _cert_pdf_path: Path | None = None
         _st_txt_path: Path | None = None
         _report_txt_path: Path | None = None
+        _cert_txt_path: Path | None = None
 
         def __init__(
             self,
             st_download_ok: bool = False,
             report_download_ok: bool = False,
+            cert_download_ok: bool = False,
             st_convert_garbage: bool = False,
             report_convert_garbage: bool = False,
+            cert_convert_garbage: bool = False,
             st_convert_ok: bool = False,
             report_convert_ok: bool = False,
+            cert_convert_ok: bool = False,
             st_extract_ok: bool = False,
             report_extract_ok: bool = False,
+            cert_extract_ok: bool = False,
             st_pdf_hash: str | None = None,
             report_pdf_hash: str | None = None,
+            cert_pdf_hash: str | None = None,
             st_txt_hash: str | None = None,
             report_txt_hash: str | None = None,
+            cert_txt_hash: str | None = None,
         ):
             super().__init__()
             self.st_download_ok = st_download_ok
             self.report_download_ok = report_download_ok
+            self.cert_download_ok = cert_download_ok
             self.st_convert_garbage = st_convert_garbage
             self.report_convert_garbage = report_convert_garbage
+            self.cert_convert_garbage = cert_convert_garbage
             self.st_convert_ok = st_convert_ok
             self.report_convert_ok = report_convert_ok
+            self.cert_convert_ok = cert_convert_ok
             self.st_extract_ok = st_extract_ok
             self.report_extract_ok = report_extract_ok
+            self.cert_extract_ok = cert_extract_ok
             self.st_pdf_hash = st_pdf_hash
             self.report_pdf_hash = report_pdf_hash
+            self.cert_pdf_hash = cert_pdf_hash
             self.st_txt_hash = st_txt_hash
             self.report_txt_hash = report_txt_hash
+            self.cert_txt_hash = cert_txt_hash
 
         @property
         def st_pdf_path(self) -> Path:
@@ -170,6 +190,16 @@ class CCCertificate(
         def report_pdf_path(self, pth: str | Path | None) -> None:
             self._report_pdf_path = Path(pth) if pth else None
 
+        @property
+        def cert_pdf_path(self) -> Path:
+            if not self._cert_pdf_path:
+                raise ValueError(f"cert_pdf_path not set on {type(self)}")
+            return self._cert_pdf_path
+
+        @cert_pdf_path.setter
+        def cert_pdf_path(self, pth: str | Path | None) -> None:
+            self._cert_pdf_path = Path(pth) if pth else None
+
         @property
         def st_txt_path(self) -> Path:
             if not self._st_txt_path:
@@ -190,21 +220,37 @@ class CCCertificate(
         def report_txt_path(self, pth: str | Path | None) -> None:
             self._report_txt_path = Path(pth) if pth else None
 
+        @property
+        def cert_txt_path(self) -> Path:
+            if not self._cert_txt_path:
+                raise ValueError(f"cert_txt_path not set on {type(self)}")
+            return self._cert_txt_path
+
+        @cert_txt_path.setter
+        def cert_txt_path(self, pth: str | Path | None) -> None:
+            self._cert_txt_path = Path(pth) if pth else None
+
         @property
         def serialized_attributes(self) -> list[str]:
             return [
                 "st_download_ok",
                 "report_download_ok",
+                "cert_download_ok",
                 "st_convert_garbage",
                 "report_convert_garbage",
+                "cert_convert_garbage",
                 "st_convert_ok",
                 "report_convert_ok",
+                "cert_convert_ok",
                 "st_extract_ok",
                 "report_extract_ok",
+                "cert_extract_ok",
                 "st_pdf_hash",
                 "report_pdf_hash",
+                "cert_pdf_hash",
                 "st_txt_hash",
                 "report_txt_hash",
+                "cert_txt_hash",
             ]
 
         def report_is_ok_to_download(self, fresh: bool = True) -> bool:
@@ -213,12 +259,18 @@ class CCCertificate(
         def st_is_ok_to_download(self, fresh: bool = True) -> bool:
             return True if fresh else not self.st_download_ok
 
+        def cert_is_ok_to_download(self, fresh: bool = True) -> bool:
+            return True if fresh else not self.st_download_ok
+
         def report_is_ok_to_convert(self, fresh: bool = True) -> bool:
             return self.report_download_ok if fresh else self.report_download_ok and not self.report_convert_ok
 
         def st_is_ok_to_convert(self, fresh: bool = True) -> bool:
             return self.st_download_ok if fresh else self.st_download_ok and not self.st_convert_ok
 
+        def cert_is_ok_to_convert(self, fresh: bool = True) -> bool:
+            return self.cert_download_ok if fresh else self.cert_download_ok and not self.cert_convert_ok
+
         def report_is_ok_to_analyze(self, fresh: bool = True) -> bool:
             if fresh is True:
                 return self.report_download_ok and self.report_convert_ok
@@ -231,6 +283,12 @@ class CCCertificate(
             else:
                 return self.st_download_ok and self.st_convert_ok and not self.st_extract_ok
 
+        def cert_is_ok_to_analyze(self, fresh: bool = True) -> bool:
+            if fresh is True:
+                return self.cert_download_ok and self.cert_convert_ok
+            else:
+                return self.cert_download_ok and self.cert_convert_ok and not self.cert_extract_ok
+
     @dataclass
     class PdfData(BasePdfData, ComplexSerializableType):
         """
@@ -239,12 +297,16 @@ class CCCertificate(
 
         report_metadata: dict[str, Any] | None = field(default=None)
         st_metadata: dict[str, Any] | None = field(default=None)
+        cert_metadata: dict[str, Any] | None = field(default=None)
         report_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
         st_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
+        cert_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
         report_keywords: dict[str, Any] | None = field(default=None)
         st_keywords: dict[str, Any] | None = field(default=None)
+        cert_keywords: dict[str, Any] | None = field(default=None)
         report_filename: str | None = field(default=None)
         st_filename: str | None = field(default=None)
+        cert_filename: str | None = field(default=None)
 
         def __bool__(self) -> bool:
             return any(x is not None for x in vars(self))
@@ -814,25 +876,33 @@ class CCCertificate(
         self,
         report_pdf_dir: str | Path | None,
         st_pdf_dir: str | Path | None,
+        cert_pdf_dir: str | Path | None,
         report_txt_dir: str | Path | None,
         st_txt_dir: str | Path | None,
+        cert_txt_dir: str | Path | None,
     ) -> None:
         """
         Sets paths to files given the requested directories
 
         :param Optional[Union[str, Path]] report_pdf_dir: Directory where pdf reports shall be stored
         :param Optional[Union[str, Path]] st_pdf_dir: Directory where pdf security targets shall be stored
+        :param Optional[Union[str, Path]] cert_pdf_dir: Directory where pdf certificates shall be stored
         :param Optional[Union[str, Path]] report_txt_dir: Directory where txt reports shall be stored
         :param Optional[Union[str, Path]] st_txt_dir: Directory where txt security targets shall be stored
+        :param Optional[Union[str, Path]] cert_txt_dir: Directory where txtcertificates shall be stored
         """
         if report_pdf_dir:
             self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf")
         if st_pdf_dir:
             self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf")
+        if cert_pdf_dir:
+            self.state.cert_pdf_path = Path(cert_pdf_dir) / (self.dgst + ".pdf")
         if report_txt_dir:
             self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + ".txt")
         if st_txt_dir:
             self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + ".txt")
+        if cert_txt_dir:
+            self.state.cert_txt_path = Path(cert_txt_dir) / (self.dgst + ".txt")
 
     @staticmethod
     def download_pdf_report(cert: CCCertificate) -> CCCertificate:
@@ -879,12 +949,34 @@ class CCCertificate(
             cert.pdf_data.st_filename = unquote_plus(str(urlparse(cert.st_link).path).split("/")[-1])
         return cert
 
+    @staticmethod
+    def download_pdf_cert(cert: CCCertificate) -> CCCertificate:
+        """
+        Downloads pdf of the certificate. Staticmethod to allow for parallelization.
+
+        :param CCCertificate cert: cert to download the pdf of
+        :return CCCertificate: returns the modified certificate with updated state
+        """
+        exit_code: str | int = (
+            helpers.download_file(cert.cert_link, cert.state.cert_pdf_path) if cert.cert_link else "No link"
+        )
+
+        if exit_code != requests.codes.ok:
+            error_msg = f"failed to download certificate from {cert.cert_link}, code: {exit_code}"
+            logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
+            cert.state.cert_download_ok = False
+        else:
+            cert.state.cert_download_ok = True
+            cert.state.cert_pdf_hash = helpers.get_sha256_filepath(cert.state.cert_pdf_path)
+            cert.pdf_data.cert_filename = unquote_plus(str(urlparse(cert.cert_link).path).split("/")[-1])
+        return cert
+
     @staticmethod
     def convert_report_pdf(cert: CCCertificate) -> CCCertificate:
         """
         Converts the pdf certification report to txt, given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to download the pdf report for
+        :param CCCertificate cert: cert to convert the pdf report for
         :return CCCertificate: the modified certificate with updated state
         """
         ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(
@@ -906,7 +998,7 @@ class CCCertificate(
         """
         Converts the pdf security target to txt, given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to download the pdf security target for
+        :param CCCertificate cert: cert to convert the pdf security target for
         :return CCCertificate: the modified certificate with updated state
         """
         ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path)
@@ -922,18 +1014,23 @@ class CCCertificate(
         return cert
 
     @staticmethod
-    def extract_st_pdf_metadata(cert: CCCertificate) -> CCCertificate:
+    def convert_cert_pdf(cert: CCCertificate) -> CCCertificate:
         """
-        Extracts metadata from security target pdf given the certificate. Staticmethod to allow for parallelization.
+        Converts the pdf certificate to txt, given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to extract the metadata for.
+        :param CCCertificate cert: cert to convert the certificate for
         :return CCCertificate: the modified certificate with updated state
         """
-        response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path)
-        if response != constants.RETURNCODE_OK:
-            cert.state.st_extract_ok = False
+        ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.cert_pdf_path, cert.state.cert_txt_path)
+        # If OCR was done the result was garbage
+        cert.state.cert_convert_garbage = ocr_done
+        # And put the whole result into convert_ok
+        cert.state.cert_convert_ok = ok_result
+        if not ok_result:
+            error_msg = "failed to convert security target pdf->txt"
+            logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
         else:
-            cert.state.st_extract_ok = True
+            cert.state.cert_txt_hash = helpers.get_sha256_filepath(cert.state.cert_txt_path)
         return cert
 
     @staticmethod
@@ -952,20 +1049,33 @@ class CCCertificate(
         return cert
 
     @staticmethod
-    def extract_st_pdf_frontpage(cert: CCCertificate) -> CCCertificate:
+    def extract_st_pdf_metadata(cert: CCCertificate) -> CCCertificate:
         """
-        Extracts data from security target pdf frontpage given the certificate. Staticmethod to allow for parallelization.
+        Extracts metadata from security target pdf given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to extract the frontpage data for.
+        :param CCCertificate cert: cert to extract the metadata for.
         :return CCCertificate: the modified certificate with updated state
         """
-        cert.pdf_data.st_frontpage = {}
+        response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.st_extract_ok = False
+        else:
+            cert.state.st_extract_ok = True
+        return cert
 
-        for header_type, associated_header_func in HEADERS.items():
-            response, cert.pdf_data.st_frontpage[header_type] = associated_header_func(cert.state.st_txt_path)
+    @staticmethod
+    def extract_cert_pdf_metadata(cert: CCCertificate) -> CCCertificate:
+        """
+        Extracts metadata from certificate pdf given the certificate. Staticmethod to allow for parallelization.
 
-            if response != constants.RETURNCODE_OK:
-                cert.state.st_extract_ok = False
+        :param CCCertificate cert: cert to extract the metadata for.
+        :return CCCertificate: the modified certificate with updated state
+        """
+        response, cert.pdf_data.cert_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.cert_pdf_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.cert_extract_ok = False
+        else:
+            cert.state.cert_extract_ok = True
         return cert
 
     @staticmethod
@@ -988,7 +1098,7 @@ class CCCertificate(
     @staticmethod
     def extract_report_pdf_keywords(cert: CCCertificate) -> CCCertificate:
         """
-        Matches regular expresions in txt obtained from certification report and extracts the matches into attribute.
+        Matches regular expressions in txt obtained from certification report and extracts the matches into attribute.
         Static method to allow for parallelization
 
         :param CCCertificate cert: certificate to extract the keywords for.
@@ -1004,7 +1114,7 @@ class CCCertificate(
     @staticmethod
     def extract_st_pdf_keywords(cert: CCCertificate) -> CCCertificate:
         """
-        Matches regular expresions in txt obtained from security target and extracts the matches into attribute.
+        Matches regular expressions in txt obtained from security target and extracts the matches into attribute.
         Static method to allow for parallelization
 
         :param CCCertificate cert: certificate to extract the keywords for.
@@ -1017,6 +1127,22 @@ class CCCertificate(
             cert.pdf_data.st_keywords = st_keywords
         return cert
 
+    @staticmethod
+    def extract_cert_pdf_keywords(cert: CCCertificate) -> CCCertificate:
+        """
+        Matches regular expressions in txt obtained from the certificate and extracts the matches into attribute.
+        Static method to allow for parallelization
+
+        :param CCCertificate cert: certificate to extract the keywords for.
+        :return CCCertificate: the modified certificate with extracted keywords.
+        """
+        cert_keywords = sec_certs.utils.extract.extract_keywords(cert.state.cert_txt_path, cc_rules)
+        if cert_keywords is None:
+            cert.state.cert_extract_ok = False
+        else:
+            cert.pdf_data.cert_keywords = cert_keywords
+        return cert
+
     def compute_heuristics_version(self) -> None:
         """
         Fills in the heuristically obtained version of certified product into attribute in heuristics class.
diff --git a/tests/cc/test_cc_certificate.py b/tests/cc/test_cc_certificate.py
index bdc10f32..32bd3a59 100644
--- a/tests/cc/test_cc_certificate.py
+++ b/tests/cc/test_cc_certificate.py
@@ -42,10 +42,6 @@ def test_extract_metadata(vulnerable_certificate: CCCertificate):
 
 
 def test_extract_frontpage(vulnerable_certificate: CCCertificate):
-    vulnerable_certificate.state.st_extract_ok = True
-    CCCertificate.extract_st_pdf_frontpage(vulnerable_certificate)
-    assert vulnerable_certificate.state.st_extract_ok
-
     vulnerable_certificate.state.report_extract_ok = True
     CCCertificate.extract_report_pdf_frontpage(vulnerable_certificate)
     assert vulnerable_certificate.state.report_extract_ok
diff --git a/tests/data/cc/certificate/fictional_cert.json b/tests/data/cc/certificate/fictional_cert.json
index ff95795c..96f42c07 100644
--- a/tests/data/cc/certificate/fictional_cert.json
+++ b/tests/data/cc/certificate/fictional_cert.json
@@ -43,27 +43,37 @@
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "st_download_ok": false,
         "report_download_ok": false,
+        "cert_download_ok": false,
         "st_convert_garbage": false,
         "report_convert_garbage": false,
+        "cert_convert_garbage": false,
         "st_convert_ok": false,
         "report_convert_ok": false,
+        "cert_convert_ok": false,
         "st_extract_ok": false,
         "report_extract_ok": false,
+        "cert_extract_ok": false,
         "st_pdf_hash": null,
         "report_pdf_hash": null,
+        "cert_pdf_hash": null,
         "st_txt_hash": null,
-        "report_txt_hash": null
+        "report_txt_hash": null,
+        "cert_txt_hash": null
     },
     "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
         "report_metadata": null,
         "st_metadata": null,
+        "cert_metadata": null,
         "report_frontpage": null,
         "st_frontpage": null,
+        "cert_frontpage": null,
         "report_keywords": null,
         "st_keywords": null,
+        "cert_keywords": null,
         "report_filename": null,
-        "st_filename": null
+        "st_filename": null,
+        "cert_filename": null
     },
     "heuristics": {
         "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
diff --git a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
index 4c1ff4c7..bc720a76 100644
--- a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
+++ b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
@@ -24,27 +24,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": true,
                 "report_download_ok": true,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": "d42e4364d037ba742fcd4050a9a84d0e6300f93eb68bcfe8c61f72c429c9ceca",
                 "report_pdf_hash": "80bada65614c1b037c13efa78996a8910700d0e05a3ca217286f76d7dacefe62",
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": "383-7-159 MR v1.0e.pdf",
-                "st_filename": "383-7-159 ST v1.4 CCRA.pdf"
+                "st_filename": "383-7-159 ST v1.4 CCRA.pdf",
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json
index e32cffa4..e60ecf2c 100644
--- a/tests/data/cc/dataset/toy_dataset.json
+++ b/tests/data/cc/dataset/toy_dataset.json
@@ -47,27 +47,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": false,
                 "report_download_ok": false,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": null,
                 "report_pdf_hash": null,
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
@@ -136,27 +146,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": false,
                 "report_download_ok": false,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": null,
                 "report_pdf_hash": null,
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
@@ -233,27 +253,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": false,
                 "report_download_ok": false,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": null,
                 "report_pdf_hash": null,
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
-- 
cgit v1.3.1


From bb3b54d177797726d473f624be83c91266d612e9 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Tue, 13 Feb 2024 17:11:58 +0100
Subject: Refactor document state in CC.

---
 src/sec_certs/dataset/cc.py                        |  28 +-
 src/sec_certs/model/cc_matching.py                 |   4 +-
 src/sec_certs/sample/cc.py                         | 342 +++++++--------------
 tests/cc/test_cc_dataset.py                        |  32 +-
 tests/cc/test_cc_maintenance_updates.py            |   6 +-
 tests/data/cc/analysis/cc_full_dataset.json        |  39 ++-
 tests/data/cc/analysis/reference_dataset.json      | 111 +++++--
 .../analysis/transitive_vulnerability_dataset.json | 111 +++++--
 tests/data/cc/analysis/vulnerable_dataset.json     |  78 ++++-
 tests/data/cc/certificate/fictional_cert.json      |  45 +--
 .../maintenances/maintenance_updates.json          |  45 +--
 tests/data/cc/dataset/toy_dataset.json             | 135 ++++----
 12 files changed, 530 insertions(+), 446 deletions(-)

diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index 30a6038a..d622a588 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -564,7 +564,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Downloading PDFs of CC certification reports.")
     def _download_reports(self, fresh: bool = True) -> None:
         self.reports_pdf_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh) and x.report_link]
+        certs_to_process = [x for x in self if x.state.report.is_ok_to_download(fresh) and x.report_link]
 
         if not fresh and certs_to_process:
             logger.info(
@@ -580,7 +580,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Downloading PDFs of CC security targets.")
     def _download_targets(self, fresh: bool = True) -> None:
         self.targets_pdf_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.st_is_ok_to_download(fresh)]
+        certs_to_process = [x for x in self if x.state.st.is_ok_to_download(fresh)]
 
         if not fresh and certs_to_process:
             logger.info(
@@ -596,7 +596,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Downloading PDFs of CC certificates.")
     def _download_certs(self, fresh: bool = True) -> None:
         self.certificates_pdf_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.cert_is_ok_to_download(fresh)]
+        certs_to_process = [x for x in self if x.state.cert.is_ok_to_download(fresh)]
 
         if not fresh and certs_to_process:
             logger.info(
@@ -612,7 +612,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Converting PDFs of certification reports to txt.")
     def _convert_reports_to_txt(self, fresh: bool = True) -> None:
         self.reports_txt_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_convert(fresh)]
+        certs_to_process = [x for x in self if x.state.report.is_ok_to_convert(fresh)]
 
         if not fresh and certs_to_process:
             logger.info(
@@ -628,7 +628,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Converting PDFs of security targets to txt.")
     def _convert_targets_to_txt(self, fresh: bool = True) -> None:
         self.targets_txt_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.st_is_ok_to_convert(fresh)]
+        certs_to_process = [x for x in self if x.state.st.is_ok_to_convert(fresh)]
 
         if fresh:
             logger.info("Converting PDFs of security targets to txt.")
@@ -646,7 +646,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Converting PDFs of certificates to txt.")
     def _convert_certs_to_txt(self, fresh: bool = True) -> None:
         self.certificates_txt_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.cert_is_ok_to_convert(fresh)]
+        certs_to_process = [x for x in self if x.state.cert.is_ok_to_convert(fresh)]
 
         if fresh:
             logger.info("Converting PDFs of certificates to txt.")
@@ -668,7 +668,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Extracting report metadata")
     def _extract_report_metadata(self) -> None:
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.report.is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_metadata,
             certs_to_process,
@@ -679,7 +679,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Extracting target metadata")
     def _extract_target_metadata(self) -> None:
-        certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.st.is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_metadata,
             certs_to_process,
@@ -690,7 +690,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Extracting cert metadata")
     def _extract_cert_metadata(self) -> None:
-        certs_to_process = [x for x in self if x.state.cert_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.cert.is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_cert_pdf_metadata,
             certs_to_process,
@@ -705,7 +705,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Extracting report frontpages")
     def _extract_report_frontpage(self) -> None:
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.report.is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_frontpage,
             certs_to_process,
@@ -720,7 +720,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Extracting report keywords")
     def _extract_report_keywords(self) -> None:
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.report.is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_report_pdf_keywords,
             certs_to_process,
@@ -731,7 +731,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Extracting target keywords")
     def _extract_target_keywords(self) -> None:
-        certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.st.is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_st_pdf_keywords,
             certs_to_process,
@@ -742,7 +742,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Extracting cert keywords")
     def _extract_cert_keywords(self) -> None:
-        certs_to_process = [x for x in self if x.state.cert_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.cert.is_ok_to_analyze()]
         processed_certs = cert_processing.process_parallel(
             CCCertificate.extract_cert_pdf_keywords,
             certs_to_process,
@@ -764,7 +764,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
 
     @staged(logger, "Computing heuristics: Deriving information about laboratories involved in certification.")
     def _compute_cert_labs(self) -> None:
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
+        certs_to_process = [x for x in self if x.state.report.is_ok_to_analyze()]
         for cert in certs_to_process:
             cert.compute_heuristics_cert_lab()
 
diff --git a/src/sec_certs/model/cc_matching.py b/src/sec_certs/model/cc_matching.py
index 4e6f5735..21d94b74 100644
--- a/src/sec_certs/model/cc_matching.py
+++ b/src/sec_certs/model/cc_matching.py
@@ -75,10 +75,10 @@ class CCSchemeMatcher(AbstractMatcher[CCCertificate]):
         if self._product == cert.name and self._vendor == cert.manufacturer:
             return 99
         # If we match the report hash, return early.
-        if cert.state.report_pdf_hash == self._report_hash and self._report_hash is not None:
+        if cert.state.report.pdf_hash == self._report_hash and self._report_hash is not None:
             return 95
         # If we match the target hash, return early.
-        if cert.state.st_pdf_hash == self._target_hash and self._target_hash is not None:
+        if cert.state.st.pdf_hash == self._target_hash and self._target_hash is not None:
             return 93
 
         # Fuzzy match at the end with some penalization.
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index a4426d01..42cb15d4 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -5,7 +5,6 @@ import re
 from collections import Counter, defaultdict
 from dataclasses import dataclass, field
 from datetime import date, datetime
-from enum import Enum
 from pathlib import Path
 from typing import Any, ClassVar
 from urllib.parse import unquote_plus, urlparse
@@ -39,9 +38,61 @@ HEADERS = {
 }
 
 
-class ReferenceType(Enum):
-    DIRECT = "direct"
-    INDIRECT = "indirect"
+@dataclass
+class CCDocumentState(ComplexSerializableType):
+    download_ok: bool = False  # Whether download went OK
+    convert_garbage: bool = False  # Whether initial conversion resulted in garbage
+    convert_ok: bool = False  # Whether overall conversion went OK (either pdftotext or via OCR)
+    extract_ok: bool = False  # Whether extraction went OK
+
+    pdf_hash: str | None = None
+    txt_hash: str | None = None
+
+    _pdf_path: Path | None = None
+    _txt_path: Path | None = None
+
+    def is_ok_to_download(self, fresh: bool = True) -> bool:
+        return True if fresh else not self.download_ok
+
+    def is_ok_to_convert(self, fresh: bool = True) -> bool:
+        return self.download_ok if fresh else self.download_ok and not self.convert_ok
+
+    def is_ok_to_analyze(self, fresh: bool = True) -> bool:
+        if fresh:
+            return self.download_ok and self.convert_ok
+        else:
+            return self.download_ok and self.convert_ok and not self.extract_ok
+
+    @property
+    def pdf_path(self) -> Path:
+        if not self._pdf_path:
+            raise ValueError(f"pdf_path not set on {type(self)}")
+        return self._pdf_path
+
+    @pdf_path.setter
+    def pdf_path(self, pth: str | Path | None) -> None:
+        self._pdf_path = Path(pth) if pth else None
+
+    @property
+    def txt_path(self) -> Path:
+        if not self._txt_path:
+            raise ValueError(f"txt_path not set on {type(self)}")
+        return self._txt_path
+
+    @txt_path.setter
+    def txt_path(self, pth: str | Path | None) -> None:
+        self._txt_path = Path(pth) if pth else None
+
+    @property
+    def serialized_attributes(self) -> list[str]:
+        return [
+            "download_ok",
+            "convert_garbage",
+            "convert_ok",
+            "extract_ok",
+            "pdf_hash",
+            "txt_hash",
+        ]
 
 
 class CCCertificate(
@@ -95,199 +146,20 @@ class CCCertificate(
         def __lt__(self, other):
             return self.maintenance_date < other.maintenance_date
 
-    @dataclass(init=False)
+    @dataclass
     class InternalState(ComplexSerializableType):
         """
         Holds internal state of the certificate, whether downloads and converts of individual components succeeded. Also
         holds information about errors and paths to the files.
         """
 
-        st_download_ok: bool  # Whether target download went OK
-        report_download_ok: bool  # Whether report download went OK
-        cert_download_ok: bool  # Whether certificate download went OK
-        st_convert_garbage: bool  # Whether initial target conversion resulted in garbage
-        report_convert_garbage: bool  # Whether initial report conversion resulted in garbage
-        cert_convert_garbage: bool  # Whether initial certificate conversion resulted in garbage
-        st_convert_ok: bool  # Whether overall target conversion went OK (either pdftotext or via OCR)
-        report_convert_ok: bool  # Whether overall report conversion went OK (either pdftotext or via OCR)
-        cert_convert_ok: bool  # Whether overall certificate conversion went OK (either pdftotext or via OCR)
-        st_extract_ok: bool  # Whether target extraction went OK
-        report_extract_ok: bool  # Whether report extraction went OK
-        cert_extract_ok: bool  # Whether certificate extraction went OK
-
-        st_pdf_hash: str | None
-        report_pdf_hash: str | None
-        cert_pdf_hash: str | None
-        st_txt_hash: str | None
-        report_txt_hash: str | None
-        cert_txt_hash: str | None
-
-        _st_pdf_path: Path | None = None
-        _report_pdf_path: Path | None = None
-        _cert_pdf_path: Path | None = None
-        _st_txt_path: Path | None = None
-        _report_txt_path: Path | None = None
-        _cert_txt_path: Path | None = None
-
-        def __init__(
-            self,
-            st_download_ok: bool = False,
-            report_download_ok: bool = False,
-            cert_download_ok: bool = False,
-            st_convert_garbage: bool = False,
-            report_convert_garbage: bool = False,
-            cert_convert_garbage: bool = False,
-            st_convert_ok: bool = False,
-            report_convert_ok: bool = False,
-            cert_convert_ok: bool = False,
-            st_extract_ok: bool = False,
-            report_extract_ok: bool = False,
-            cert_extract_ok: bool = False,
-            st_pdf_hash: str | None = None,
-            report_pdf_hash: str | None = None,
-            cert_pdf_hash: str | None = None,
-            st_txt_hash: str | None = None,
-            report_txt_hash: str | None = None,
-            cert_txt_hash: str | None = None,
-        ):
-            super().__init__()
-            self.st_download_ok = st_download_ok
-            self.report_download_ok = report_download_ok
-            self.cert_download_ok = cert_download_ok
-            self.st_convert_garbage = st_convert_garbage
-            self.report_convert_garbage = report_convert_garbage
-            self.cert_convert_garbage = cert_convert_garbage
-            self.st_convert_ok = st_convert_ok
-            self.report_convert_ok = report_convert_ok
-            self.cert_convert_ok = cert_convert_ok
-            self.st_extract_ok = st_extract_ok
-            self.report_extract_ok = report_extract_ok
-            self.cert_extract_ok = cert_extract_ok
-            self.st_pdf_hash = st_pdf_hash
-            self.report_pdf_hash = report_pdf_hash
-            self.cert_pdf_hash = cert_pdf_hash
-            self.st_txt_hash = st_txt_hash
-            self.report_txt_hash = report_txt_hash
-            self.cert_txt_hash = cert_txt_hash
-
-        @property
-        def st_pdf_path(self) -> Path:
-            if not self._st_pdf_path:
-                raise ValueError(f"st_pdf_path not set on {type(self)}")
-            return self._st_pdf_path
-
-        @st_pdf_path.setter
-        def st_pdf_path(self, pth: str | Path | None) -> None:
-            self._st_pdf_path = Path(pth) if pth else None
-
-        @property
-        def report_pdf_path(self) -> Path:
-            if not self._report_pdf_path:
-                raise ValueError(f"report_pdf_path not set on {type(self)}")
-            return self._report_pdf_path
-
-        @report_pdf_path.setter
-        def report_pdf_path(self, pth: str | Path | None) -> None:
-            self._report_pdf_path = Path(pth) if pth else None
-
-        @property
-        def cert_pdf_path(self) -> Path:
-            if not self._cert_pdf_path:
-                raise ValueError(f"cert_pdf_path not set on {type(self)}")
-            return self._cert_pdf_path
-
-        @cert_pdf_path.setter
-        def cert_pdf_path(self, pth: str | Path | None) -> None:
-            self._cert_pdf_path = Path(pth) if pth else None
-
-        @property
-        def st_txt_path(self) -> Path:
-            if not self._st_txt_path:
-                raise ValueError(f"st_txt_path not set on {type(self)}")
-            return self._st_txt_path
-
-        @st_txt_path.setter
-        def st_txt_path(self, pth: str | Path | None) -> None:
-            self._st_txt_path = Path(pth) if pth else None
-
-        @property
-        def report_txt_path(self) -> Path:
-            if not self._report_txt_path:
-                raise ValueError(f"report_txt_path not set on {type(self)}")
-            return self._report_txt_path
-
-        @report_txt_path.setter
-        def report_txt_path(self, pth: str | Path | None) -> None:
-            self._report_txt_path = Path(pth) if pth else None
-
-        @property
-        def cert_txt_path(self) -> Path:
-            if not self._cert_txt_path:
-                raise ValueError(f"cert_txt_path not set on {type(self)}")
-            return self._cert_txt_path
-
-        @cert_txt_path.setter
-        def cert_txt_path(self, pth: str | Path | None) -> None:
-            self._cert_txt_path = Path(pth) if pth else None
+        report: CCDocumentState = field(default_factory=CCDocumentState)
+        st: CCDocumentState = field(default_factory=CCDocumentState)
+        cert: CCDocumentState = field(default_factory=CCDocumentState)
 
         @property
         def serialized_attributes(self) -> list[str]:
-            return [
-                "st_download_ok",
-                "report_download_ok",
-                "cert_download_ok",
-                "st_convert_garbage",
-                "report_convert_garbage",
-                "cert_convert_garbage",
-                "st_convert_ok",
-                "report_convert_ok",
-                "cert_convert_ok",
-                "st_extract_ok",
-                "report_extract_ok",
-                "cert_extract_ok",
-                "st_pdf_hash",
-                "report_pdf_hash",
-                "cert_pdf_hash",
-                "st_txt_hash",
-                "report_txt_hash",
-                "cert_txt_hash",
-            ]
-
-        def report_is_ok_to_download(self, fresh: bool = True) -> bool:
-            return True if fresh else not self.report_download_ok
-
-        def st_is_ok_to_download(self, fresh: bool = True) -> bool:
-            return True if fresh else not self.st_download_ok
-
-        def cert_is_ok_to_download(self, fresh: bool = True) -> bool:
-            return True if fresh else not self.st_download_ok
-
-        def report_is_ok_to_convert(self, fresh: bool = True) -> bool:
-            return self.report_download_ok if fresh else self.report_download_ok and not self.report_convert_ok
-
-        def st_is_ok_to_convert(self, fresh: bool = True) -> bool:
-            return self.st_download_ok if fresh else self.st_download_ok and not self.st_convert_ok
-
-        def cert_is_ok_to_convert(self, fresh: bool = True) -> bool:
-            return self.cert_download_ok if fresh else self.cert_download_ok and not self.cert_convert_ok
-
-        def report_is_ok_to_analyze(self, fresh: bool = True) -> bool:
-            if fresh is True:
-                return self.report_download_ok and self.report_convert_ok
-            else:
-                return self.report_download_ok and self.report_convert_ok and not self.report_extract_ok
-
-        def st_is_ok_to_analyze(self, fresh: bool = True) -> bool:
-            if fresh is True:
-                return self.st_download_ok and self.st_convert_ok
-            else:
-                return self.st_download_ok and self.st_convert_ok and not self.st_extract_ok
-
-        def cert_is_ok_to_analyze(self, fresh: bool = True) -> bool:
-            if fresh is True:
-                return self.cert_download_ok and self.cert_convert_ok
-            else:
-                return self.cert_download_ok and self.cert_convert_ok and not self.cert_extract_ok
+            return ["report", "st", "cert"]
 
     @dataclass
     class PdfData(BasePdfData, ComplexSerializableType):
@@ -892,17 +764,17 @@ class CCCertificate(
         :param Optional[Union[str, Path]] cert_txt_dir: Directory where txtcertificates shall be stored
         """
         if report_pdf_dir:
-            self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf")
+            self.state.report.pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf")
         if st_pdf_dir:
-            self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf")
+            self.state.st.pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf")
         if cert_pdf_dir:
-            self.state.cert_pdf_path = Path(cert_pdf_dir) / (self.dgst + ".pdf")
+            self.state.cert.pdf_path = Path(cert_pdf_dir) / (self.dgst + ".pdf")
         if report_txt_dir:
-            self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + ".txt")
+            self.state.report.txt_path = Path(report_txt_dir) / (self.dgst + ".txt")
         if st_txt_dir:
-            self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + ".txt")
+            self.state.st.txt_path = Path(st_txt_dir) / (self.dgst + ".txt")
         if cert_txt_dir:
-            self.state.cert_txt_path = Path(cert_txt_dir) / (self.dgst + ".txt")
+            self.state.cert.txt_path = Path(cert_txt_dir) / (self.dgst + ".txt")
 
     @staticmethod
     def download_pdf_report(cert: CCCertificate) -> CCCertificate:
@@ -916,14 +788,14 @@ class CCCertificate(
         if not cert.report_link:
             exit_code = "No link"
         else:
-            exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path)
+            exit_code = helpers.download_file(cert.report_link, cert.state.report.pdf_path)
         if exit_code != requests.codes.ok:
             error_msg = f"failed to download report from {cert.report_link}, code: {exit_code}"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
-            cert.state.report_download_ok = False
+            cert.state.report.download_ok = False
         else:
-            cert.state.report_download_ok = True
-            cert.state.report_pdf_hash = helpers.get_sha256_filepath(cert.state.report_pdf_path)
+            cert.state.report.download_ok = True
+            cert.state.report.pdf_hash = helpers.get_sha256_filepath(cert.state.report.pdf_path)
             cert.pdf_data.report_filename = unquote_plus(str(urlparse(cert.report_link).path).split("/")[-1])
         return cert
 
@@ -936,16 +808,16 @@ class CCCertificate(
         :return CCCertificate: returns the modified certificate with updated state
         """
         exit_code: str | int = (
-            helpers.download_file(cert.st_link, cert.state.st_pdf_path) if cert.st_link else "No link"
+            helpers.download_file(cert.st_link, cert.state.st.pdf_path) if cert.st_link else "No link"
         )
 
         if exit_code != requests.codes.ok:
             error_msg = f"failed to download ST from {cert.st_link}, code: {exit_code}"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
-            cert.state.st_download_ok = False
+            cert.state.st.download_ok = False
         else:
-            cert.state.st_download_ok = True
-            cert.state.st_pdf_hash = helpers.get_sha256_filepath(cert.state.st_pdf_path)
+            cert.state.st.download_ok = True
+            cert.state.st.pdf_hash = helpers.get_sha256_filepath(cert.state.st.pdf_path)
             cert.pdf_data.st_filename = unquote_plus(str(urlparse(cert.st_link).path).split("/")[-1])
         return cert
 
@@ -958,16 +830,16 @@ class CCCertificate(
         :return CCCertificate: returns the modified certificate with updated state
         """
         exit_code: str | int = (
-            helpers.download_file(cert.cert_link, cert.state.cert_pdf_path) if cert.cert_link else "No link"
+            helpers.download_file(cert.cert_link, cert.state.cert.pdf_path) if cert.cert_link else "No link"
         )
 
         if exit_code != requests.codes.ok:
             error_msg = f"failed to download certificate from {cert.cert_link}, code: {exit_code}"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
-            cert.state.cert_download_ok = False
+            cert.state.cert.download_ok = False
         else:
-            cert.state.cert_download_ok = True
-            cert.state.cert_pdf_hash = helpers.get_sha256_filepath(cert.state.cert_pdf_path)
+            cert.state.cert.download_ok = True
+            cert.state.cert.pdf_hash = helpers.get_sha256_filepath(cert.state.cert.pdf_path)
             cert.pdf_data.cert_filename = unquote_plus(str(urlparse(cert.cert_link).path).split("/")[-1])
         return cert
 
@@ -980,17 +852,17 @@ class CCCertificate(
         :return CCCertificate: the modified certificate with updated state
         """
         ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(
-            cert.state.report_pdf_path, cert.state.report_txt_path
+            cert.state.report.pdf_path, cert.state.report.txt_path
         )
         # If OCR was done the result was garbage
-        cert.state.report_convert_garbage = ocr_done
+        cert.state.report.convert_garbage = ocr_done
         # And put the whole result into convert_ok
-        cert.state.report_convert_ok = ok_result
+        cert.state.report.convert_ok = ok_result
         if not ok_result:
             error_msg = "failed to convert report pdf->txt"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
         else:
-            cert.state.report_txt_hash = helpers.get_sha256_filepath(cert.state.report_txt_path)
+            cert.state.report.txt_hash = helpers.get_sha256_filepath(cert.state.report.txt_path)
         return cert
 
     @staticmethod
@@ -1001,16 +873,16 @@ class CCCertificate(
         :param CCCertificate cert: cert to convert the pdf security target for
         :return CCCertificate: the modified certificate with updated state
         """
-        ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path)
+        ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.st.pdf_path, cert.state.st.txt_path)
         # If OCR was done the result was garbage
-        cert.state.st_convert_garbage = ocr_done
+        cert.state.st.convert_garbage = ocr_done
         # And put the whole result into convert_ok
-        cert.state.st_convert_ok = ok_result
+        cert.state.st.convert_ok = ok_result
         if not ok_result:
             error_msg = "failed to convert security target pdf->txt"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
         else:
-            cert.state.st_txt_hash = helpers.get_sha256_filepath(cert.state.st_txt_path)
+            cert.state.st.txt_hash = helpers.get_sha256_filepath(cert.state.st.txt_path)
         return cert
 
     @staticmethod
@@ -1021,16 +893,16 @@ class CCCertificate(
         :param CCCertificate cert: cert to convert the certificate for
         :return CCCertificate: the modified certificate with updated state
         """
-        ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.cert_pdf_path, cert.state.cert_txt_path)
+        ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.cert.pdf_path, cert.state.cert.txt_path)
         # If OCR was done the result was garbage
-        cert.state.cert_convert_garbage = ocr_done
+        cert.state.cert.convert_garbage = ocr_done
         # And put the whole result into convert_ok
-        cert.state.cert_convert_ok = ok_result
+        cert.state.cert.convert_ok = ok_result
         if not ok_result:
             error_msg = "failed to convert security target pdf->txt"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
         else:
-            cert.state.cert_txt_hash = helpers.get_sha256_filepath(cert.state.cert_txt_path)
+            cert.state.cert.txt_hash = helpers.get_sha256_filepath(cert.state.cert.txt_path)
         return cert
 
     @staticmethod
@@ -1041,11 +913,11 @@ class CCCertificate(
         :param CCCertificate cert: cert to extract the metadata for.
         :return CCCertificate: the modified certificate with updated state
         """
-        response, cert.pdf_data.report_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.report_pdf_path)
+        response, cert.pdf_data.report_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.report.pdf_path)
         if response != constants.RETURNCODE_OK:
-            cert.state.report_extract_ok = False
+            cert.state.report.extract_ok = False
         else:
-            cert.state.report_extract_ok = True
+            cert.state.report.extract_ok = True
         return cert
 
     @staticmethod
@@ -1056,11 +928,11 @@ class CCCertificate(
         :param CCCertificate cert: cert to extract the metadata for.
         :return CCCertificate: the modified certificate with updated state
         """
-        response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path)
+        response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st.pdf_path)
         if response != constants.RETURNCODE_OK:
-            cert.state.st_extract_ok = False
+            cert.state.st.extract_ok = False
         else:
-            cert.state.st_extract_ok = True
+            cert.state.st.extract_ok = True
         return cert
 
     @staticmethod
@@ -1071,11 +943,11 @@ class CCCertificate(
         :param CCCertificate cert: cert to extract the metadata for.
         :return CCCertificate: the modified certificate with updated state
         """
-        response, cert.pdf_data.cert_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.cert_pdf_path)
+        response, cert.pdf_data.cert_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.cert.pdf_path)
         if response != constants.RETURNCODE_OK:
-            cert.state.cert_extract_ok = False
+            cert.state.cert.extract_ok = False
         else:
-            cert.state.cert_extract_ok = True
+            cert.state.cert.extract_ok = True
         return cert
 
     @staticmethod
@@ -1089,10 +961,10 @@ class CCCertificate(
         cert.pdf_data.report_frontpage = {}
 
         for header_type, associated_header_func in HEADERS.items():
-            response, cert.pdf_data.report_frontpage[header_type] = associated_header_func(cert.state.report_txt_path)
+            response, cert.pdf_data.report_frontpage[header_type] = associated_header_func(cert.state.report.txt_path)
 
             if response != constants.RETURNCODE_OK:
-                cert.state.report_extract_ok = False
+                cert.state.report.extract_ok = False
         return cert
 
     @staticmethod
@@ -1104,9 +976,9 @@ class CCCertificate(
         :param CCCertificate cert: certificate to extract the keywords for.
         :return CCCertificate: the modified certificate with extracted keywords.
         """
-        report_keywords = sec_certs.utils.extract.extract_keywords(cert.state.report_txt_path, cc_rules)
+        report_keywords = sec_certs.utils.extract.extract_keywords(cert.state.report.txt_path, cc_rules)
         if report_keywords is None:
-            cert.state.report_extract_ok = False
+            cert.state.report.extract_ok = False
         else:
             cert.pdf_data.report_keywords = report_keywords
         return cert
@@ -1120,9 +992,9 @@ class CCCertificate(
         :param CCCertificate cert: certificate to extract the keywords for.
         :return CCCertificate: the modified certificate with extracted keywords.
         """
-        st_keywords = sec_certs.utils.extract.extract_keywords(cert.state.st_txt_path, cc_rules)
+        st_keywords = sec_certs.utils.extract.extract_keywords(cert.state.st.txt_path, cc_rules)
         if st_keywords is None:
-            cert.state.st_extract_ok = False
+            cert.state.st.extract_ok = False
         else:
             cert.pdf_data.st_keywords = st_keywords
         return cert
@@ -1136,9 +1008,9 @@ class CCCertificate(
         :param CCCertificate cert: certificate to extract the keywords for.
         :return CCCertificate: the modified certificate with extracted keywords.
         """
-        cert_keywords = sec_certs.utils.extract.extract_keywords(cert.state.cert_txt_path, cc_rules)
+        cert_keywords = sec_certs.utils.extract.extract_keywords(cert.state.cert.txt_path, cc_rules)
         if cert_keywords is None:
-            cert.state.cert_extract_ok = False
+            cert.state.cert.extract_ok = False
         else:
             cert.pdf_data.cert_keywords = cert_keywords
         return cert
diff --git a/tests/cc/test_cc_dataset.py b/tests/cc/test_cc_dataset.py
index 0701e678..4fe40a40 100644
--- a/tests/cc/test_cc_dataset.py
+++ b/tests/cc/test_cc_dataset.py
@@ -28,36 +28,36 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path):
         toy_dataset.download_all_artifacts()
 
         if not (
-            toy_dataset["309ac2fd7f2dcf17"].state.report_download_ok
-            or toy_dataset["309ac2fd7f2dcf17"].state.st_download_ok
-            or toy_dataset["8cf86948f02f047d"].state.report_download_ok
-            or toy_dataset["8cf86948f02f047d"].state.st_download_ok
-            or toy_dataset["8a5e6bcda602920c"].state.report_download_ok
-            or toy_dataset["8a5e6bcda602920c"].state.st_download_ok
+            toy_dataset["309ac2fd7f2dcf17"].state.report.download_ok
+            or toy_dataset["309ac2fd7f2dcf17"].state.st.download_ok
+            or toy_dataset["8cf86948f02f047d"].state.report.download_ok
+            or toy_dataset["8cf86948f02f047d"].state.st.download_ok
+            or toy_dataset["8a5e6bcda602920c"].state.report.download_ok
+            or toy_dataset["8a5e6bcda602920c"].state.st.download_ok
         ):
             pytest.xfail(reason="Fail due to error during download")
 
         toy_dataset.convert_all_pdfs()
 
         for cert in toy_dataset:
-            assert cert.state.report_pdf_hash == template_report_pdf_hashes[cert.dgst]
-            assert cert.state.st_pdf_hash == template_st_pdf_hashes[cert.dgst]
-            assert not cert.state.report_convert_garbage
-            assert not cert.state.st_convert_garbage
-            assert cert.state.report_convert_ok
-            assert cert.state.st_convert_ok
-            assert cert.state.report_txt_path.exists()
-            assert cert.state.st_txt_path.exists()
+            assert cert.state.report.pdf_hash == template_report_pdf_hashes[cert.dgst]
+            assert cert.state.st.pdf_hash == template_st_pdf_hashes[cert.dgst]
+            assert not cert.state.report.convert_garbage
+            assert not cert.state.st.convert_garbage
+            assert cert.state.report.convert_ok
+            assert cert.state.st.convert_ok
+            assert cert.state.report.txt_path.exists()
+            assert cert.state.st.txt_path.exists()
 
         template_report_txt_path = data_dir / "report_309ac2fd7f2dcf17.txt"
         template_st_txt_path = data_dir / "target_309ac2fd7f2dcf17.txt"
         assert (
-            abs(toy_dataset["309ac2fd7f2dcf17"].state.st_txt_path.stat().st_size - template_st_txt_path.stat().st_size)
+            abs(toy_dataset["309ac2fd7f2dcf17"].state.st.txt_path.stat().st_size - template_st_txt_path.stat().st_size)
             < 1000
         )
         assert (
             abs(
-                toy_dataset["309ac2fd7f2dcf17"].state.report_txt_path.stat().st_size
+                toy_dataset["309ac2fd7f2dcf17"].state.report.txt_path.stat().st_size
                 - template_report_txt_path.stat().st_size
             )
             < 1000
diff --git a/tests/cc/test_cc_maintenance_updates.py b/tests/cc/test_cc_maintenance_updates.py
index 1144d3e0..4e94d980 100644
--- a/tests/cc/test_cc_maintenance_updates.py
+++ b/tests/cc/test_cc_maintenance_updates.py
@@ -43,11 +43,11 @@ def test_download_artifacts(mu_dset: CCDatasetMaintenanceUpdates):
     mu_dset.download_all_artifacts()
     mu = mu_dset["cert_8a5e6bcda602920c_update_559ed93dd80320b5"]
 
-    if not (mu.state.report_download_ok or mu.state.st_download_ok):
+    if not (mu.state.report.download_ok or mu.state.st.download_ok):
         pytest.xfail(reason="Fail due to error on CC server.")
 
-    assert mu.state.report_pdf_hash == "80bada65614c1b037c13efa78996a8910700d0e05a3ca217286f76d7dacefe62"
-    assert mu.state.st_pdf_hash == "d42e4364d037ba742fcd4050a9a84d0e6300f93eb68bcfe8c61f72c429c9ceca"
+    assert mu.state.report.pdf_hash == "80bada65614c1b037c13efa78996a8910700d0e05a3ca217286f76d7dacefe62"
+    assert mu.state.st.pdf_hash == "d42e4364d037ba742fcd4050a9a84d0e6300f93eb68bcfe8c61f72c429c9ceca"
 
 
 def test_dataset_to_json(mu_dset: CCDatasetMaintenanceUpdates, data_dir: Path, tmp_path: Path):
diff --git a/tests/data/cc/analysis/cc_full_dataset.json b/tests/data/cc/analysis/cc_full_dataset.json
index 3b07f9c5..ebc5241e 100644
--- a/tests/data/cc/analysis/cc_full_dataset.json
+++ b/tests/data/cc/analysis/cc_full_dataset.json
@@ -55,18 +55,33 @@
             },
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "st_download_ok": true,
-                "report_download_ok": true,
-                "st_convert_garbage": false,
-                "report_convert_garbage": false,
-                "st_convert_ok": true,
-                "report_convert_ok": true,
-                "st_extract_ok": true,
-                "report_extract_ok": true,
-                "st_pdf_hash": "f4ab8c62a2325cc634eef164a9707d2375f2e3d5c5ab9791b7c00a6034e64b62",
-                "report_pdf_hash": "1954f42e857c02a156caf8fea7abd36ee0a5953fe0e319c4ec749af277fcfb4b",
-                "st_txt_hash": "c8b4c5667a3f60edc845051e5a31a2d17b9d9a11df9e56dd89681d25e727a622",
-                "report_txt_hash": "35627594d3806ac3926ec47f466503fe27781533da12beb6f8705882fccf125e"
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": "1954f42e857c02a156caf8fea7abd36ee0a5953fe0e319c4ec749af277fcfb4b",
+                    "txt_hash": "35627594d3806ac3926ec47f466503fe27781533da12beb6f8705882fccf125e"
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": "f4ab8c62a2325cc634eef164a9707d2375f2e3d5c5ab9791b7c00a6034e64b62",
+                    "txt_hash": "c8b4c5667a3f60edc845051e5a31a2d17b9d9a11df9e56dd89681d25e727a622"
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
diff --git a/tests/data/cc/analysis/reference_dataset.json b/tests/data/cc/analysis/reference_dataset.json
index 00ab6674..4b5d7d19 100644
--- a/tests/data/cc/analysis/reference_dataset.json
+++ b/tests/data/cc/analysis/reference_dataset.json
@@ -44,16 +44,33 @@
       },
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-        "st_download_ok": true,
-        "report_download_ok": true,
-        "st_convert_ok": true,
-        "report_convert_ok": true,
-        "st_extract_ok": true,
-        "report_extract_ok": true,
-        "st_pdf_hash": "16f1c7e32517d22f6062edf439f8c67eb0d202fadaaf1c54c1f96af7f74ad7ac",
-        "report_pdf_hash": "2dc5ff15361621bb2bd7db79491b88a70049abe4ffbe8370de87f9c51f42fb50",
-        "st_txt_hash": "81c53d1e5b1c2fcb129ce1053d13cd1308f7a556921f0b9024cedf75c6b2efb7",
-        "report_txt_hash": "460e8010dbc8f5de5b87bf96fd45c71cfd9f3869f34ca6ac1ab02cbd70d2523f"
+        "report": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "2dc5ff15361621bb2bd7db79491b88a70049abe4ffbe8370de87f9c51f42fb50",
+            "txt_hash": "460e8010dbc8f5de5b87bf96fd45c71cfd9f3869f34ca6ac1ab02cbd70d2523f"
+        },
+        "st": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "16f1c7e32517d22f6062edf439f8c67eb0d202fadaaf1c54c1f96af7f74ad7ac",
+            "txt_hash": "81c53d1e5b1c2fcb129ce1053d13cd1308f7a556921f0b9024cedf75c6b2efb7"
+        },
+        "cert": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": null,
+            "txt_hash": null
+        }
       },
       "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
@@ -604,16 +621,33 @@
       },
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-        "st_download_ok": true,
-        "report_download_ok": true,
-        "st_convert_ok": true,
-        "report_convert_ok": true,
-        "st_extract_ok": true,
-        "report_extract_ok": true,
-        "st_pdf_hash": "8f288bc6133855bba6b78ccaeff86b46b4ce1db86aa1b5141c1073a74e6d13fd",
-        "report_pdf_hash": "d77dffc3f2e4d95d6103de31b7ebbec54551c3b93f01415971ec3b9f5cb33e4f",
-        "st_txt_hash": "926668bea7c427a4fcf82857bfc63420f3597b6bff39699927a58f335620eaac",
-        "report_txt_hash": "0535df1c56fb4f87153cbffee51ba4d77fac47a6f17f024aa7d9df461028bc65"
+        "report": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "d77dffc3f2e4d95d6103de31b7ebbec54551c3b93f01415971ec3b9f5cb33e4f",
+            "txt_hash": "0535df1c56fb4f87153cbffee51ba4d77fac47a6f17f024aa7d9df461028bc65"
+        },
+        "st": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "8f288bc6133855bba6b78ccaeff86b46b4ce1db86aa1b5141c1073a74e6d13fd",
+            "txt_hash": "926668bea7c427a4fcf82857bfc63420f3597b6bff39699927a58f335620eaac"
+        },
+        "cert": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": null,
+            "txt_hash": null
+        }
       },
       "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
@@ -1242,16 +1276,33 @@
       },
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-        "st_download_ok": true,
-        "report_download_ok": true,
-        "st_convert_ok": true,
-        "report_convert_ok": true,
-        "st_extract_ok": true,
-        "report_extract_ok": true,
-        "st_pdf_hash": "c72b032f6119e6dd64270296713d5626b1473bf4925018ce4dd106d76953213f",
-        "report_pdf_hash": "eca6c3a665d8bd20394cf002bff0a8a1b451337f60c8184d458ffd5b31491085",
-        "st_txt_hash": "179b07b4fc7402066a884edea494b28e324315108a5e0820184031f2e2062ad5",
-        "report_txt_hash": "11e1262fd8f5df1b140f5e8813883b71447503781399427b35adbbecd00b4d63"
+        "report": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "eca6c3a665d8bd20394cf002bff0a8a1b451337f60c8184d458ffd5b31491085",
+            "txt_hash": "11e1262fd8f5df1b140f5e8813883b71447503781399427b35adbbecd00b4d63"
+        },
+        "st": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "c72b032f6119e6dd64270296713d5626b1473bf4925018ce4dd106d76953213f",
+            "txt_hash": "179b07b4fc7402066a884edea494b28e324315108a5e0820184031f2e2062ad5"
+        },
+        "cert": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": null,
+            "txt_hash": null
+        }
       },
       "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
diff --git a/tests/data/cc/analysis/transitive_vulnerability_dataset.json b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
index fb45efde..0b995ce7 100644
--- a/tests/data/cc/analysis/transitive_vulnerability_dataset.json
+++ b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
@@ -54,16 +54,33 @@
       },
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-        "st_download_ok": true,
-        "report_download_ok": true,
-        "st_convert_ok": true,
-        "report_convert_ok": true,
-        "st_extract_ok": true,
-        "report_extract_ok": true,
-        "st_pdf_hash": "5bb93f7f3f08f30ba41abb003a2f8ce2609c385af82b863fbc0b19bd0c21a701",
-        "report_pdf_hash": "ee1ebada2c91c5a000c8d112e9e3742d09cad4c920d3f3e2a9beb01f16c69bb6",
-        "st_txt_hash": "66271d8bf0b581a2f189301438f2aee13ff3da0bb0bb180bcf518261eb695496",
-        "report_txt_hash": "9d360141a98e764b15855f519b456c4e4639f993c4f8b5ab67e9c8ae7fbfc9e4"
+        "report": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "ee1ebada2c91c5a000c8d112e9e3742d09cad4c920d3f3e2a9beb01f16c69bb6",
+            "txt_hash": "9d360141a98e764b15855f519b456c4e4639f993c4f8b5ab67e9c8ae7fbfc9e4"
+        },
+        "st": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "5bb93f7f3f08f30ba41abb003a2f8ce2609c385af82b863fbc0b19bd0c21a701",
+            "txt_hash": "66271d8bf0b581a2f189301438f2aee13ff3da0bb0bb180bcf518261eb695496"
+        },
+        "cert": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": null,
+            "txt_hash": null
+        }
       },
       "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
@@ -1360,16 +1377,33 @@
       },
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-        "st_download_ok": true,
-        "report_download_ok": true,
-        "st_convert_ok": true,
-        "report_convert_ok": true,
-        "st_extract_ok": true,
-        "report_extract_ok": true,
-        "st_pdf_hash": "4879ac5fdf9569ad88089df0122acd1c8a8f7252ed8f36aace14bbb0b805b758",
-        "report_pdf_hash": "63e6ac157e08ed37f9861458c66c015663e17fb8936746d7ae487963bdd455c7",
-        "st_txt_hash": "f7f7b8f31dddde3f0756cde8843061f01b606bdf266eca71dbcc56b3672d1db5",
-        "report_txt_hash": "dd120ba7667c2385839c96ee70c56f2a4d464fc95e3ea2818d31b3347d06fd4f"
+        "report": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "63e6ac157e08ed37f9861458c66c015663e17fb8936746d7ae487963bdd455c7",
+            "txt_hash": "dd120ba7667c2385839c96ee70c56f2a4d464fc95e3ea2818d31b3347d06fd4f"
+        },
+        "st": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "4879ac5fdf9569ad88089df0122acd1c8a8f7252ed8f36aace14bbb0b805b758",
+            "txt_hash": "f7f7b8f31dddde3f0756cde8843061f01b606bdf266eca71dbcc56b3672d1db5"
+        },
+        "cert": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": null,
+            "txt_hash": null
+        }
       },
       "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
@@ -2311,16 +2345,33 @@
       },
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-        "st_download_ok": true,
-        "report_download_ok": true,
-        "st_convert_ok": true,
-        "report_convert_ok": true,
-        "st_extract_ok": true,
-        "report_extract_ok": true,
-        "st_pdf_hash": "1a5e4c1382e284da88d93aa5557c7728c14b3fce73d81e4cf731ab24cee9ffdd",
-        "report_pdf_hash": "f10c85194adae814703781471f3a3de713383d8a9fbf5389fc7106958a8aaf90",
-        "st_txt_hash": "90b8e48add278faea4668eccba591d3992bf782669cca1b0a63bf6f21b514cd9",
-        "report_txt_hash": "0a7c65e3d11f082c8f75aba7de0079c0b1aa5e67bb28d4635cbcaa4cd200d1c2"
+        "report": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "f10c85194adae814703781471f3a3de713383d8a9fbf5389fc7106958a8aaf90",
+            "txt_hash": "0a7c65e3d11f082c8f75aba7de0079c0b1aa5e67bb28d4635cbcaa4cd200d1c2"
+        },
+        "st": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": "1a5e4c1382e284da88d93aa5557c7728c14b3fce73d81e4cf731ab24cee9ffdd",
+            "txt_hash": "90b8e48add278faea4668eccba591d3992bf782669cca1b0a63bf6f21b514cd9"
+        },
+        "cert": {
+            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "download_ok": false,
+            "convert_garbage": false,
+            "convert_ok": false,
+            "extract_ok": false,
+            "pdf_hash": null,
+            "txt_hash": null
+        }
       },
       "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
diff --git a/tests/data/cc/analysis/vulnerable_dataset.json b/tests/data/cc/analysis/vulnerable_dataset.json
index 1b23ef0d..8e9ba284 100644
--- a/tests/data/cc/analysis/vulnerable_dataset.json
+++ b/tests/data/cc/analysis/vulnerable_dataset.json
@@ -39,23 +39,48 @@
             "maintenance_updates": [],
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "st_download_ok": true,
-                "report_download_ok": true,
-                "st_convert_ok": true,
-                "report_convert_ok": true,
-                "st_extract_ok": true,
-                "report_extract_ok": true
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": true,
+                    "convert_garbage": false,
+                    "convert_ok": true,
+                    "extract_ok": true,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": true,
+                    "convert_garbage": false,
+                    "convert_ok": true,
+                    "extract_ok": true,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
@@ -95,23 +120,48 @@
             "maintenance_updates": [],
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "st_download_ok": true,
-                "report_download_ok": true,
-                "st_convert_ok": true,
-                "report_convert_ok": true,
-                "st_extract_ok": true,
-                "report_extract_ok": true
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": true,
+                    "convert_garbage": false,
+                    "convert_ok": true,
+                    "extract_ok": true,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": true,
+                    "convert_garbage": false,
+                    "convert_ok": true,
+                    "extract_ok": true,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
diff --git a/tests/data/cc/certificate/fictional_cert.json b/tests/data/cc/certificate/fictional_cert.json
index 96f42c07..d6a7be34 100644
--- a/tests/data/cc/certificate/fictional_cert.json
+++ b/tests/data/cc/certificate/fictional_cert.json
@@ -41,24 +41,33 @@
     },
     "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-        "st_download_ok": false,
-        "report_download_ok": false,
-        "cert_download_ok": false,
-        "st_convert_garbage": false,
-        "report_convert_garbage": false,
-        "cert_convert_garbage": false,
-        "st_convert_ok": false,
-        "report_convert_ok": false,
-        "cert_convert_ok": false,
-        "st_extract_ok": false,
-        "report_extract_ok": false,
-        "cert_extract_ok": false,
-        "st_pdf_hash": null,
-        "report_pdf_hash": null,
-        "cert_pdf_hash": null,
-        "st_txt_hash": null,
-        "report_txt_hash": null,
-        "cert_txt_hash": null
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
     },
     "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
diff --git a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
index bc720a76..032cc10d 100644
--- a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
+++ b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
@@ -22,24 +22,33 @@
             "st_link": "https://www.commoncriteriaportal.org/files/epfiles/383-7-159%20ST%20v1.4%20CCRA.pdf",
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "st_download_ok": true,
-                "report_download_ok": true,
-                "cert_download_ok": false,
-                "st_convert_garbage": false,
-                "report_convert_garbage": false,
-                "cert_convert_garbage": false,
-                "st_convert_ok": false,
-                "report_convert_ok": false,
-                "cert_convert_ok": false,
-                "st_extract_ok": false,
-                "report_extract_ok": false,
-                "cert_extract_ok": false,
-                "st_pdf_hash": "d42e4364d037ba742fcd4050a9a84d0e6300f93eb68bcfe8c61f72c429c9ceca",
-                "report_pdf_hash": "80bada65614c1b037c13efa78996a8910700d0e05a3ca217286f76d7dacefe62",
-                "cert_pdf_hash": null,
-                "st_txt_hash": null,
-                "report_txt_hash": null,
-                "cert_txt_hash": null
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": true,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": "80bada65614c1b037c13efa78996a8910700d0e05a3ca217286f76d7dacefe62",
+                    "txt_hash": null
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": true,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": "d42e4364d037ba742fcd4050a9a84d0e6300f93eb68bcfe8c61f72c429c9ceca",
+                    "txt_hash": null
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json
index e60ecf2c..33aa7770 100644
--- a/tests/data/cc/dataset/toy_dataset.json
+++ b/tests/data/cc/dataset/toy_dataset.json
@@ -45,24 +45,33 @@
             },
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "st_download_ok": false,
-                "report_download_ok": false,
-                "cert_download_ok": false,
-                "st_convert_garbage": false,
-                "report_convert_garbage": false,
-                "cert_convert_garbage": false,
-                "st_convert_ok": false,
-                "report_convert_ok": false,
-                "cert_convert_ok": false,
-                "st_extract_ok": false,
-                "report_extract_ok": false,
-                "cert_extract_ok": false,
-                "st_pdf_hash": null,
-                "report_pdf_hash": null,
-                "cert_pdf_hash": null,
-                "st_txt_hash": null,
-                "report_txt_hash": null,
-                "cert_txt_hash": null
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
@@ -144,24 +153,33 @@
             },
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "st_download_ok": false,
-                "report_download_ok": false,
-                "cert_download_ok": false,
-                "st_convert_garbage": false,
-                "report_convert_garbage": false,
-                "cert_convert_garbage": false,
-                "st_convert_ok": false,
-                "report_convert_ok": false,
-                "cert_convert_ok": false,
-                "st_extract_ok": false,
-                "report_extract_ok": false,
-                "cert_extract_ok": false,
-                "st_pdf_hash": null,
-                "report_pdf_hash": null,
-                "cert_pdf_hash": null,
-                "st_txt_hash": null,
-                "report_txt_hash": null,
-                "cert_txt_hash": null
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
@@ -251,24 +269,33 @@
             },
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "st_download_ok": false,
-                "report_download_ok": false,
-                "cert_download_ok": false,
-                "st_convert_garbage": false,
-                "report_convert_garbage": false,
-                "cert_convert_garbage": false,
-                "st_convert_ok": false,
-                "report_convert_ok": false,
-                "cert_convert_ok": false,
-                "st_extract_ok": false,
-                "report_extract_ok": false,
-                "cert_extract_ok": false,
-                "st_pdf_hash": null,
-                "report_pdf_hash": null,
-                "cert_pdf_hash": null,
-                "st_txt_hash": null,
-                "report_txt_hash": null,
-                "cert_txt_hash": null
+                "report": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "st": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                },
+                "cert": {
+                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "download_ok": false,
+                    "convert_garbage": false,
+                    "convert_ok": false,
+                    "extract_ok": false,
+                    "pdf_hash": null,
+                    "txt_hash": null
+                }
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
-- 
cgit v1.3.1


From fffefed8310969c145eca27f2765712f5295558a Mon Sep 17 00:00:00 2001
From: J08nY
Date: Tue, 13 Feb 2024 17:24:00 +0100
Subject: Move CCDocumentState to cert class.

---
 .../model/references_nlp/segment_extractor.py      |   8 +-
 src/sec_certs/sample/cc.py                         | 130 +++++++++++----------
 tests/cc/test_cc_certificate.py                    |  20 ++--
 tests/data/cc/analysis/cc_full_dataset.json        |   6 +-
 tests/data/cc/analysis/reference_dataset.json      |  18 +--
 .../analysis/transitive_vulnerability_dataset.json |  18 +--
 tests/data/cc/analysis/vulnerable_dataset.json     |  12 +-
 tests/data/cc/certificate/fictional_cert.json      |   6 +-
 .../maintenances/maintenance_updates.json          |   6 +-
 tests/data/cc/dataset/toy_dataset.json             |  18 +--
 10 files changed, 126 insertions(+), 116 deletions(-)

diff --git a/src/sec_certs/model/references_nlp/segment_extractor.py b/src/sec_certs/model/references_nlp/segment_extractor.py
index d63208ec..f77c8f5b 100644
--- a/src/sec_certs/model/references_nlp/segment_extractor.py
+++ b/src/sec_certs/model/references_nlp/segment_extractor.py
@@ -173,9 +173,9 @@ class ReferenceSegmentExtractor:
         - Loads manually annotated samples
         - Combines all of that into single dataframe
         """
-        target_certs = [x for x in certs if x.heuristics.st_references.directly_referencing and x.state.st_txt_path]
+        target_certs = [x for x in certs if x.heuristics.st_references.directly_referencing and x.state.st.txt_path]
         report_certs = [
-            x for x in certs if x.heuristics.report_references.directly_referencing and x.state.report_txt_path
+            x for x in certs if x.heuristics.report_references.directly_referencing and x.state.report.txt_path
         ]
         df_targets = self._build_df(target_certs, "target")
         df_reports = self._build_df(report_certs, "report")
@@ -217,8 +217,8 @@ class ReferenceSegmentExtractor:
                 for key, val in actual_references.items()
             ]
 
-        (certs[0].state.report_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True)
-        (certs[0].state.st_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True)
+        (certs[0].state.report.txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True)
+        (certs[0].state.st.txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True)
         return list(itertools.chain.from_iterable(get_cert_records(cert, source) for cert in certs))
 
     def _build_df(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> pd.DataFrame:
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 42cb15d4..bfc0b07d 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -38,63 +38,6 @@ HEADERS = {
 }
 
 
-@dataclass
-class CCDocumentState(ComplexSerializableType):
-    download_ok: bool = False  # Whether download went OK
-    convert_garbage: bool = False  # Whether initial conversion resulted in garbage
-    convert_ok: bool = False  # Whether overall conversion went OK (either pdftotext or via OCR)
-    extract_ok: bool = False  # Whether extraction went OK
-
-    pdf_hash: str | None = None
-    txt_hash: str | None = None
-
-    _pdf_path: Path | None = None
-    _txt_path: Path | None = None
-
-    def is_ok_to_download(self, fresh: bool = True) -> bool:
-        return True if fresh else not self.download_ok
-
-    def is_ok_to_convert(self, fresh: bool = True) -> bool:
-        return self.download_ok if fresh else self.download_ok and not self.convert_ok
-
-    def is_ok_to_analyze(self, fresh: bool = True) -> bool:
-        if fresh:
-            return self.download_ok and self.convert_ok
-        else:
-            return self.download_ok and self.convert_ok and not self.extract_ok
-
-    @property
-    def pdf_path(self) -> Path:
-        if not self._pdf_path:
-            raise ValueError(f"pdf_path not set on {type(self)}")
-        return self._pdf_path
-
-    @pdf_path.setter
-    def pdf_path(self, pth: str | Path | None) -> None:
-        self._pdf_path = Path(pth) if pth else None
-
-    @property
-    def txt_path(self) -> Path:
-        if not self._txt_path:
-            raise ValueError(f"txt_path not set on {type(self)}")
-        return self._txt_path
-
-    @txt_path.setter
-    def txt_path(self, pth: str | Path | None) -> None:
-        self._txt_path = Path(pth) if pth else None
-
-    @property
-    def serialized_attributes(self) -> list[str]:
-        return [
-            "download_ok",
-            "convert_garbage",
-            "convert_ok",
-            "extract_ok",
-            "pdf_hash",
-            "txt_hash",
-        ]
-
-
 class CCCertificate(
     Certificate["CCCertificate", "CCCertificate.Heuristics", "CCCertificate.PdfData"],
     PandasSerializableType,
@@ -147,15 +90,82 @@ class CCCertificate(
             return self.maintenance_date < other.maintenance_date
 
     @dataclass
+    class DocumentState(ComplexSerializableType):
+        download_ok: bool = False  # Whether download went OK
+        convert_garbage: bool = False  # Whether initial conversion resulted in garbage
+        convert_ok: bool = False  # Whether overall conversion went OK (either pdftotext or via OCR)
+        extract_ok: bool = False  # Whether extraction went OK
+
+        pdf_hash: str | None = None
+        txt_hash: str | None = None
+
+        _pdf_path: Path | None = None
+        _txt_path: Path | None = None
+
+        def is_ok_to_download(self, fresh: bool = True) -> bool:
+            return True if fresh else not self.download_ok
+
+        def is_ok_to_convert(self, fresh: bool = True) -> bool:
+            return self.download_ok if fresh else self.download_ok and not self.convert_ok
+
+        def is_ok_to_analyze(self, fresh: bool = True) -> bool:
+            if fresh:
+                return self.download_ok and self.convert_ok
+            else:
+                return self.download_ok and self.convert_ok and not self.extract_ok
+
+        @property
+        def pdf_path(self) -> Path:
+            if not self._pdf_path:
+                raise ValueError(f"pdf_path not set on {type(self)}")
+            return self._pdf_path
+
+        @pdf_path.setter
+        def pdf_path(self, pth: str | Path | None) -> None:
+            self._pdf_path = Path(pth) if pth else None
+
+        @property
+        def txt_path(self) -> Path:
+            if not self._txt_path:
+                raise ValueError(f"txt_path not set on {type(self)}")
+            return self._txt_path
+
+        @txt_path.setter
+        def txt_path(self, pth: str | Path | None) -> None:
+            self._txt_path = Path(pth) if pth else None
+
+        @property
+        def serialized_attributes(self) -> list[str]:
+            return [
+                "download_ok",
+                "convert_garbage",
+                "convert_ok",
+                "extract_ok",
+                "pdf_hash",
+                "txt_hash",
+            ]
+
+    @dataclass(init=False)
     class InternalState(ComplexSerializableType):
         """
         Holds internal state of the certificate, whether downloads and converts of individual components succeeded. Also
         holds information about errors and paths to the files.
         """
 
-        report: CCDocumentState = field(default_factory=CCDocumentState)
-        st: CCDocumentState = field(default_factory=CCDocumentState)
-        cert: CCDocumentState = field(default_factory=CCDocumentState)
+        report: CCCertificate.DocumentState
+        st: CCCertificate.DocumentState
+        cert: CCCertificate.DocumentState
+
+        def __init__(
+            self,
+            report: CCCertificate.DocumentState | None = None,
+            st: CCCertificate.DocumentState | None = None,
+            cert: CCCertificate.DocumentState | None = None,
+        ):
+            super().__init__()
+            self.report = report if report is not None else CCCertificate.DocumentState()
+            self.st = st if st is not None else CCCertificate.DocumentState()
+            self.cert = cert if cert is not None else CCCertificate.DocumentState()
 
         @property
         def serialized_attributes(self) -> list[str]:
diff --git a/tests/cc/test_cc_certificate.py b/tests/cc/test_cc_certificate.py
index 32bd3a59..f90245e0 100644
--- a/tests/cc/test_cc_certificate.py
+++ b/tests/cc/test_cc_certificate.py
@@ -32,29 +32,29 @@ def vulnerable_certificate(tmp_path_factory) -> CCCertificate:
 
 
 def test_extract_metadata(vulnerable_certificate: CCCertificate):
-    vulnerable_certificate.state.st_extract_ok = True
+    vulnerable_certificate.state.st.extract_ok = True
     CCCertificate.extract_st_pdf_metadata(vulnerable_certificate)
-    assert vulnerable_certificate.state.st_extract_ok
+    assert vulnerable_certificate.state.st.extract_ok
 
-    vulnerable_certificate.state.report_extract_ok = True
+    vulnerable_certificate.state.report.extract_ok = True
     CCCertificate.extract_report_pdf_metadata(vulnerable_certificate)
-    assert vulnerable_certificate.state.report_extract_ok
+    assert vulnerable_certificate.state.report.extract_ok
 
 
 def test_extract_frontpage(vulnerable_certificate: CCCertificate):
-    vulnerable_certificate.state.report_extract_ok = True
+    vulnerable_certificate.state.report.extract_ok = True
     CCCertificate.extract_report_pdf_frontpage(vulnerable_certificate)
-    assert vulnerable_certificate.state.report_extract_ok
+    assert vulnerable_certificate.state.report.extract_ok
 
 
 def test_keyword_extraction(vulnerable_certificate: CCCertificate):
-    vulnerable_certificate.state.st_extract_ok = True
+    vulnerable_certificate.state.st.extract_ok = True
     CCCertificate.extract_st_pdf_keywords(vulnerable_certificate)
-    assert vulnerable_certificate.state.st_extract_ok
+    assert vulnerable_certificate.state.st.extract_ok
 
-    vulnerable_certificate.state.report_extract_ok = True
+    vulnerable_certificate.state.report.extract_ok = True
     CCCertificate.extract_report_pdf_keywords(vulnerable_certificate)
-    assert vulnerable_certificate.state.report_extract_ok
+    assert vulnerable_certificate.state.report.extract_ok
 
 
 def test_cert_link_escaping(cert_one: CCCertificate):
diff --git a/tests/data/cc/analysis/cc_full_dataset.json b/tests/data/cc/analysis/cc_full_dataset.json
index ebc5241e..4baf4758 100644
--- a/tests/data/cc/analysis/cc_full_dataset.json
+++ b/tests/data/cc/analysis/cc_full_dataset.json
@@ -56,7 +56,7 @@
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -65,7 +65,7 @@
                     "txt_hash": "35627594d3806ac3926ec47f466503fe27781533da12beb6f8705882fccf125e"
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -74,7 +74,7 @@
                     "txt_hash": "c8b4c5667a3f60edc845051e5a31a2d17b9d9a11df9e56dd89681d25e727a622"
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
diff --git a/tests/data/cc/analysis/reference_dataset.json b/tests/data/cc/analysis/reference_dataset.json
index 4b5d7d19..b0ef6683 100644
--- a/tests/data/cc/analysis/reference_dataset.json
+++ b/tests/data/cc/analysis/reference_dataset.json
@@ -45,7 +45,7 @@
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "report": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -54,7 +54,7 @@
             "txt_hash": "460e8010dbc8f5de5b87bf96fd45c71cfd9f3869f34ca6ac1ab02cbd70d2523f"
         },
         "st": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -63,7 +63,7 @@
             "txt_hash": "81c53d1e5b1c2fcb129ce1053d13cd1308f7a556921f0b9024cedf75c6b2efb7"
         },
         "cert": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -622,7 +622,7 @@
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "report": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -631,7 +631,7 @@
             "txt_hash": "0535df1c56fb4f87153cbffee51ba4d77fac47a6f17f024aa7d9df461028bc65"
         },
         "st": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -640,7 +640,7 @@
             "txt_hash": "926668bea7c427a4fcf82857bfc63420f3597b6bff39699927a58f335620eaac"
         },
         "cert": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -1277,7 +1277,7 @@
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "report": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -1286,7 +1286,7 @@
             "txt_hash": "11e1262fd8f5df1b140f5e8813883b71447503781399427b35adbbecd00b4d63"
         },
         "st": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -1295,7 +1295,7 @@
             "txt_hash": "179b07b4fc7402066a884edea494b28e324315108a5e0820184031f2e2062ad5"
         },
         "cert": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
diff --git a/tests/data/cc/analysis/transitive_vulnerability_dataset.json b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
index 0b995ce7..21391a12 100644
--- a/tests/data/cc/analysis/transitive_vulnerability_dataset.json
+++ b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
@@ -55,7 +55,7 @@
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "report": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -64,7 +64,7 @@
             "txt_hash": "9d360141a98e764b15855f519b456c4e4639f993c4f8b5ab67e9c8ae7fbfc9e4"
         },
         "st": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -73,7 +73,7 @@
             "txt_hash": "66271d8bf0b581a2f189301438f2aee13ff3da0bb0bb180bcf518261eb695496"
         },
         "cert": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -1378,7 +1378,7 @@
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "report": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -1387,7 +1387,7 @@
             "txt_hash": "dd120ba7667c2385839c96ee70c56f2a4d464fc95e3ea2818d31b3347d06fd4f"
         },
         "st": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -1396,7 +1396,7 @@
             "txt_hash": "f7f7b8f31dddde3f0756cde8843061f01b606bdf266eca71dbcc56b3672d1db5"
         },
         "cert": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -2346,7 +2346,7 @@
       "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "report": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -2355,7 +2355,7 @@
             "txt_hash": "0a7c65e3d11f082c8f75aba7de0079c0b1aa5e67bb28d4635cbcaa4cd200d1c2"
         },
         "st": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
@@ -2364,7 +2364,7 @@
             "txt_hash": "90b8e48add278faea4668eccba591d3992bf782669cca1b0a63bf6f21b514cd9"
         },
         "cert": {
-            "_type": "sec_certs.sample.cc.CCDocumentState",
+            "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
             "download_ok": false,
             "convert_garbage": false,
             "convert_ok": false,
diff --git a/tests/data/cc/analysis/vulnerable_dataset.json b/tests/data/cc/analysis/vulnerable_dataset.json
index 8e9ba284..7978cb5a 100644
--- a/tests/data/cc/analysis/vulnerable_dataset.json
+++ b/tests/data/cc/analysis/vulnerable_dataset.json
@@ -40,7 +40,7 @@
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": true,
                     "convert_garbage": false,
                     "convert_ok": true,
@@ -49,7 +49,7 @@
                     "txt_hash": null
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": true,
                     "convert_garbage": false,
                     "convert_ok": true,
@@ -58,7 +58,7 @@
                     "txt_hash": null
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -121,7 +121,7 @@
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": true,
                     "convert_garbage": false,
                     "convert_ok": true,
@@ -130,7 +130,7 @@
                     "txt_hash": null
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": true,
                     "convert_garbage": false,
                     "convert_ok": true,
@@ -139,7 +139,7 @@
                     "txt_hash": null
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
diff --git a/tests/data/cc/certificate/fictional_cert.json b/tests/data/cc/certificate/fictional_cert.json
index d6a7be34..54781dd5 100644
--- a/tests/data/cc/certificate/fictional_cert.json
+++ b/tests/data/cc/certificate/fictional_cert.json
@@ -42,7 +42,7 @@
     "state": {
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -51,7 +51,7 @@
                     "txt_hash": null
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -60,7 +60,7 @@
                     "txt_hash": null
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
diff --git a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
index 032cc10d..5596b597 100644
--- a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
+++ b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
@@ -23,7 +23,7 @@
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": true,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -32,7 +32,7 @@
                     "txt_hash": null
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": true,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -41,7 +41,7 @@
                     "txt_hash": null
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json
index 33aa7770..dc70bc4e 100644
--- a/tests/data/cc/dataset/toy_dataset.json
+++ b/tests/data/cc/dataset/toy_dataset.json
@@ -46,7 +46,7 @@
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -55,7 +55,7 @@
                     "txt_hash": null
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -64,7 +64,7 @@
                     "txt_hash": null
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -154,7 +154,7 @@
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -163,7 +163,7 @@
                     "txt_hash": null
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -172,7 +172,7 @@
                     "txt_hash": null
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -270,7 +270,7 @@
             "state": {
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "report": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -279,7 +279,7 @@
                     "txt_hash": null
                 },
                 "st": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
@@ -288,7 +288,7 @@
                     "txt_hash": null
                 },
                 "cert": {
-                    "_type": "sec_certs.sample.cc.CCDocumentState",
+                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
                     "download_ok": false,
                     "convert_garbage": false,
                     "convert_ok": false,
-- 
cgit v1.3.1


From 05afc93e597564f802d36ef56ccdbf8108967dd2 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Tue, 13 Feb 2024 17:51:37 +0100
Subject: Add cert_id extraction from certificate files.

---
 src/sec_certs/sample/cc.py | 110 +++++++++++++++++++++++++--------------------
 1 file changed, 61 insertions(+), 49 deletions(-)

diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index bfc0b07d..bb4b0c7c 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -277,52 +277,59 @@ class CCCertificate(
 
         def filename_cert_id(self, scheme: str) -> dict[str, float]:
             """
-            Get cert_id candidates from the matches in the report filename.
+            Get cert_id candidates from the matches in the report filename and cert filename.
             """
-            if not self.report_filename:
-                return {}
             scheme_filename_rules = rules["cc_filename_cert_id"][scheme]
             if not scheme_filename_rules:
                 return {}
             scheme_meta = schemes[scheme]
-            matches: Counter = Counter()
-            for rule in scheme_filename_rules:
-                match = re.search(rule, self.report_filename)
-                if match:
-                    try:
-                        meta = match.groupdict()
-                        cert_id = scheme_meta(meta)
-                        matches[cert_id] += 1
-                    except Exception:
-                        continue
-            if not matches:
-                return {}
-            total = max(matches.values())
-            results = {}
-            for candidate, count in matches.items():
-                results[candidate] = count / total
+            results: dict[str, float] = {}
+            for fname in (self.report_filename, self.cert_filename):
+                if not fname:
+                    continue
+
+                matches: Counter = Counter()
+                for rule in scheme_filename_rules:
+                    match = re.search(rule, fname)
+                    if match:
+                        try:
+                            meta = match.groupdict()
+                            cert_id = scheme_meta(meta)
+                            matches[cert_id] += 1
+                        except Exception:
+                            continue
+                if not matches:
+                    continue
+                total = max(matches.values())
+
+                for candidate, count in matches.items():
+                    results.setdefault(candidate, 0)
+                    results[candidate] += count / total
             # TODO count length in weight
             return results
 
         def keywords_cert_id(self, scheme: str) -> dict[str, float]:
             """
-            Get cert_id candidates from the keywords matches in the report.
+            Get cert_id candidates from the keywords matches in the report and cert.
             """
-            if not self.report_keywords:
-                return {}
-            cert_id_matches = self.report_keywords.get("cc_cert_id")
-            if not cert_id_matches:
-                return {}
+            results: dict[str, float] = {}
+            for keywords in (self.report_keywords, self.cert_keywords):
+                if not keywords:
+                    continue
+                cert_id_matches = keywords.get("cc_cert_id")
+                if not cert_id_matches:
+                    continue
 
-            if scheme not in cert_id_matches:
-                return {}
-            matches: Counter = Counter(cert_id_matches[scheme])
-            if not matches:
-                return {}
-            total = max(matches.values())
-            results = {}
-            for candidate, count in matches.items():
-                results[candidate] = count / total
+                if scheme not in cert_id_matches:
+                    continue
+                matches: Counter = Counter(cert_id_matches[scheme])
+                if not matches:
+                    continue
+                total = max(matches.values())
+
+                for candidate, count in matches.items():
+                    results.setdefault(candidate, 0)
+                    results[candidate] += count / total
             # TODO count length in weight
             return results
 
@@ -332,22 +339,27 @@ class CCCertificate(
             """
             scheme_rules = rules["cc_cert_id"][scheme]
             fields = ("/Title", "/Subject")
-            matches: Counter = Counter()
-            for meta_field in fields:
-                field_val = self.report_metadata.get(meta_field) if self.report_metadata else None
-                if not field_val:
+            results: dict[str, float] = {}
+            for metadata in (self.report_metadata, self.cert_metadata):
+                if not metadata:
                     continue
-                for rule in scheme_rules:
-                    match = re.search(rule, field_val)
-                    if match:
-                        cert_id = normalize_match_string(match.group())
-                        matches[cert_id] += 1
-            if not matches:
-                return {}
-            total = max(matches.values())
-            results = {}
-            for candidate, count in matches.items():
-                results[candidate] = count / total
+                matches: Counter = Counter()
+                for meta_field in fields:
+                    field_val = metadata.get(meta_field)
+                    if not field_val:
+                        continue
+                    for rule in scheme_rules:
+                        match = re.search(rule, field_val)
+                        if match:
+                            cert_id = normalize_match_string(match.group())
+                            matches[cert_id] += 1
+                if not matches:
+                    continue
+                total = max(matches.values())
+
+                for candidate, count in matches.items():
+                    results.setdefault(candidate, 0)
+                    results[candidate] += count / total
             # TODO count length in weight
             return results
 
-- 
cgit v1.3.1


From d73ddbcdfcd0768c9e594f627947eac87f1d561b Mon Sep 17 00:00:00 2001
From: J08nY
Date: Wed, 14 Feb 2024 12:58:21 +0100
Subject: Add tests for cert data extraction.

---
 src/sec_certs/dataset/cc.py            |   3 +-
 tests/cc/test_cc_dataset.py            |  11 +
 tests/data/cc/dataset/toy_dataset.json | 678 ++++++++++++++++-----------------
 3 files changed, 352 insertions(+), 340 deletions(-)

diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index d622a588..44c56afa 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -658,7 +658,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         cert_processing.process_parallel(
             CCCertificate.convert_cert_pdf,
             certs_to_process,
-            progress_bar_desc="Converting PDFs of security targets to txt",
+            progress_bar_desc="Converting PDFs of certificates to txt",
         )
 
     def _convert_all_pdfs_body(self, fresh: bool = True) -> None:
@@ -702,6 +702,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     def _extract_pdf_metadata(self) -> None:
         self._extract_report_metadata()
         self._extract_target_metadata()
+        self._extract_cert_metadata()
 
     @staged(logger, "Extracting report frontpages")
     def _extract_report_frontpage(self) -> None:
diff --git a/tests/cc/test_cc_dataset.py b/tests/cc/test_cc_dataset.py
index 4fe40a40..9d1e4022 100644
--- a/tests/cc/test_cc_dataset.py
+++ b/tests/cc/test_cc_dataset.py
@@ -23,6 +23,12 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path):
         "8a5e6bcda602920c": "fcee91f09bb72a6526a1f94d0ab754a6db3fbe3ba5773cd372df19788bb25292",
     }
 
+    template_cert_pdf_hashes = {
+        "309ac2fd7f2dcf17": "9d38bca310c4d349cc39471e0b75d939cc275db9a75b07b8a365d719cfbedcc5",
+        "8cf86948f02f047d": None,
+        "8a5e6bcda602920c": "4ba78f26f505819183256ca5a6b404fa90c750fe160c41791e4c400f64e2f6d5",
+    }
+
     with TemporaryDirectory() as td:
         toy_dataset.copy_dataset(td)
         toy_dataset.download_all_artifacts()
@@ -30,10 +36,12 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path):
         if not (
             toy_dataset["309ac2fd7f2dcf17"].state.report.download_ok
             or toy_dataset["309ac2fd7f2dcf17"].state.st.download_ok
+            or toy_dataset["309ac2fd7f2dcf17"].state.cert.download_ok
             or toy_dataset["8cf86948f02f047d"].state.report.download_ok
             or toy_dataset["8cf86948f02f047d"].state.st.download_ok
             or toy_dataset["8a5e6bcda602920c"].state.report.download_ok
             or toy_dataset["8a5e6bcda602920c"].state.st.download_ok
+            or toy_dataset["8a5e6bcda602920c"].state.cert.download_ok
         ):
             pytest.xfail(reason="Fail due to error during download")
 
@@ -42,12 +50,15 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path):
         for cert in toy_dataset:
             assert cert.state.report.pdf_hash == template_report_pdf_hashes[cert.dgst]
             assert cert.state.st.pdf_hash == template_st_pdf_hashes[cert.dgst]
+            assert cert.state.cert.pdf_hash == template_cert_pdf_hashes[cert.dgst]
             assert not cert.state.report.convert_garbage
             assert not cert.state.st.convert_garbage
             assert cert.state.report.convert_ok
             assert cert.state.st.convert_ok
             assert cert.state.report.txt_path.exists()
             assert cert.state.st.txt_path.exists()
+            if cert.cert_link:
+                assert cert.state.cert.txt_path.exists()
 
         template_report_txt_path = data_dir / "report_309ac2fd7f2dcf17.txt"
         template_st_txt_path = data_dir / "target_309ac2fd7f2dcf17.txt"
diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json
index dc70bc4e..dbf75078 100644
--- a/tests/data/cc/dataset/toy_dataset.json
+++ b/tests/data/cc/dataset/toy_dataset.json
@@ -1,345 +1,345 @@
 {
-    "_type": "sec_certs.dataset.cc.CCDataset",
-    "state": {
-        "_type": "sec_certs.dataset.dataset.Dataset.DatasetInternalState",
-        "meta_sources_parsed": true,
-        "artifacts_downloaded": false,
-        "pdfs_converted": false,
-        "auxiliary_datasets_processed": false,
-        "certs_analyzed": false
+  "_type": "sec_certs.dataset.cc.CCDataset",
+  "state": {
+    "_type": "sec_certs.dataset.dataset.Dataset.DatasetInternalState",
+    "meta_sources_parsed": true,
+    "artifacts_downloaded": false,
+    "pdfs_converted": false,
+    "auxiliary_datasets_processed": false,
+    "certs_analyzed": false
+  },
+  "timestamp": "2020-11-16 17:04:14.770153",
+  "sha256_digest": "not implemented",
+  "name": "toy dataset",
+  "description": "toy dataset description",
+  "n_certs": 3,
+  "certs": [
+    {
+      "_type": "sec_certs.sample.cc.CCCertificate",
+      "dgst": "309ac2fd7f2dcf17",
+      "status": "active",
+      "category": "Access Control Devices and Systems",
+      "name": "NetIQ Identity Manager 4.7",
+      "manufacturer": "NetIQ Corporation",
+      "scheme": "SE",
+      "security_level": {
+        "_type": "Set",
+        "elements": [
+          "ALC_FLR.2",
+          "EAL3+"
+        ]
+      },
+      "not_valid_before": "2020-06-15",
+      "not_valid_after": "2025-06-15",
+      "report_link": "https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf",
+      "st_link": "https://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf",
+      "cert_link": "https://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
+      "manufacturer_web": "https://www.netiq.com/",
+      "protection_profiles": {
+        "_type": "Set",
+        "elements": []
+      },
+      "maintenance_updates": {
+        "_type": "Set",
+        "elements": []
+      },
+      "state": {
+        "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
+        "report": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
+        },
+        "st": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
+        },
+        "cert": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
+        }
+      },
+      "pdf_data": {
+        "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
+        "report_metadata": null,
+        "st_metadata": null,
+        "cert_metadata": null,
+        "report_frontpage": null,
+        "st_frontpage": null,
+        "cert_frontpage": null,
+        "report_keywords": null,
+        "st_keywords": null,
+        "cert_keywords": null,
+        "report_filename": null,
+        "st_filename": null,
+        "cert_filename": null
+      },
+      "heuristics": {
+        "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
+        "extracted_versions": null,
+        "cpe_matches": null,
+        "verified_cpe_matches": null,
+        "related_cves": null,
+        "cert_lab": null,
+        "cert_id": null,
+        "annotated_references": null,
+        "extracted_sars": null,
+        "direct_transitive_cves": null,
+        "indirect_transitive_cves": null,
+        "report_references": {
+          "_type": "sec_certs.sample.certificate.References",
+          "directly_referenced_by": null,
+          "directly_referencing": null,
+          "indirectly_referenced_by": null,
+          "indirectly_referencing": null
+        },
+        "st_references": {
+          "_type": "sec_certs.sample.certificate.References",
+          "directly_referenced_by": null,
+          "directly_referencing": null,
+          "indirectly_referenced_by": null,
+          "indirectly_referencing": null
+        },
+        "scheme_data": null
+      }
     },
-    "timestamp": "2020-11-16 17:04:14.770153",
-    "sha256_digest": "not implemented",
-    "name": "toy dataset",
-    "description": "toy dataset description",
-    "n_certs": 3,
-    "certs": [
-        {
-            "_type": "sec_certs.sample.cc.CCCertificate",
-            "dgst": "309ac2fd7f2dcf17",
-            "status": "active",
-            "category": "Access Control Devices and Systems",
-            "name": "NetIQ Identity Manager 4.7",
-            "manufacturer": "NetIQ Corporation",
-            "scheme": "SE",
-            "security_level": {
-                "_type": "Set",
-                "elements": [
-                    "ALC_FLR.2",
-                    "EAL3+"
-                ]
-            },
-            "not_valid_before": "2020-06-15",
-            "not_valid_after": "2025-06-15",
-            "report_link": "https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf",
-            "st_link": "https://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf",
-            "cert_link": "https://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
-            "manufacturer_web": "https://www.netiq.com/",
-            "protection_profiles": {
-                "_type": "Set",
-                "elements": []
-            },
-            "maintenance_updates": {
-                "_type": "Set",
-                "elements": []
-            },
-            "state": {
-                "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "report": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                },
-                "st": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                },
-                "cert": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                }
-            },
-            "pdf_data": {
-                "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
-                "report_metadata": null,
-                "st_metadata": null,
-                "cert_metadata": null,
-                "report_frontpage": null,
-                "st_frontpage": null,
-                "cert_frontpage": null,
-                "report_keywords": null,
-                "st_keywords": null,
-                "cert_keywords": null,
-                "report_filename": null,
-                "st_filename": null,
-                "cert_filename": null
-            },
-            "heuristics": {
-                "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
-                "extracted_versions": null,
-                "cpe_matches": null,
-                "verified_cpe_matches": null,
-                "related_cves": null,
-                "cert_lab": null,
-                "cert_id": null,
-                "annotated_references": null,
-                "extracted_sars": null,
-                "direct_transitive_cves": null,
-                "indirect_transitive_cves": null,
-                "report_references": {
-                    "_type": "sec_certs.sample.certificate.References",
-                    "directly_referenced_by": null,
-                    "directly_referencing": null,
-                    "indirectly_referenced_by": null,
-                    "indirectly_referencing": null
-                },
-                "st_references": {
-                    "_type": "sec_certs.sample.certificate.References",
-                    "directly_referenced_by": null,
-                    "directly_referencing": null,
-                    "indirectly_referenced_by": null,
-                    "indirectly_referencing": null
-                },
-                "scheme_data": null
-            }
+    {
+      "_type": "sec_certs.sample.cc.CCCertificate",
+      "dgst": "8cf86948f02f047d",
+      "status": "active",
+      "category": "Access Control Devices and Systems",
+      "name": "Magic SSO V4.0",
+      "manufacturer": "Dreamsecurity Co., Ltd.",
+      "scheme": "KR",
+      "security_level": {
+        "_type": "Set",
+        "elements": []
+      },
+      "not_valid_before": "2019-11-15",
+      "not_valid_after": "2024-11-15",
+      "report_link": "https://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf",
+      "st_link": "https://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf",
+      "cert_link": null,
+      "manufacturer_web": "https://www.dreamsecurity.com/",
+      "protection_profiles": {
+        "_type": "Set",
+        "elements": [
+          {
+            "_type": "sec_certs.sample.protection_profile.ProtectionProfile",
+            "pp_name": "Korean National Protection Profile for Single Sign On V1.0",
+            "pp_eal": "EAL1+",
+            "pp_link": "https://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf",
+            "pp_ids": null
+          }
+        ]
+      },
+      "maintenance_updates": {
+        "_type": "Set",
+        "elements": []
+      },
+      "state": {
+        "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
+        "report": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
         },
-        {
-            "_type": "sec_certs.sample.cc.CCCertificate",
-            "dgst": "8cf86948f02f047d",
-            "status": "active",
-            "category": "Access Control Devices and Systems",
-            "name": "Magic SSO V4.0",
-            "manufacturer": "Dreamsecurity Co., Ltd.",
-            "scheme": "KR",
-            "security_level": {
-                "_type": "Set",
-                "elements": []
-            },
-            "not_valid_before": "2019-11-15",
-            "not_valid_after": "2024-11-15",
-            "report_link": "https://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf",
-            "st_link": "https://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf",
-            "cert_link": null,
-            "manufacturer_web": "https://www.dreamsecurity.com/",
-            "protection_profiles": {
-                "_type": "Set",
-                "elements": [
-                    {
-                        "_type": "sec_certs.sample.protection_profile.ProtectionProfile",
-                        "pp_name": "Korean National Protection Profile for Single Sign On V1.0",
-                        "pp_eal": "EAL1+",
-                        "pp_link": "https://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf",
-                        "pp_ids": null
-                    }
-                ]
-            },
-            "maintenance_updates": {
-                "_type": "Set",
-                "elements": []
-            },
-            "state": {
-                "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "report": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                },
-                "st": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                },
-                "cert": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                }
-            },
-            "pdf_data": {
-                "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
-                "report_metadata": null,
-                "st_metadata": null,
-                "cert_metadata": null,
-                "report_frontpage": null,
-                "st_frontpage": null,
-                "cert_frontpage": null,
-                "report_keywords": null,
-                "st_keywords": null,
-                "cert_keywords": null,
-                "report_filename": null,
-                "st_filename": null,
-                "cert_filename": null
-            },
-            "heuristics": {
-                "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
-                "extracted_versions": null,
-                "cpe_matches": null,
-                "verified_cpe_matches": null,
-                "related_cves": null,
-                "cert_lab": null,
-                "cert_id": null,
-                "annotated_references": null,
-                "extracted_sars": null,
-                "direct_transitive_cves": null,
-                "indirect_transitive_cves": null,
-                "report_references": {
-                    "_type": "sec_certs.sample.certificate.References",
-                    "directly_referenced_by": null,
-                    "directly_referencing": null,
-                    "indirectly_referenced_by": null,
-                    "indirectly_referencing": null
-                },
-                "st_references": {
-                    "_type": "sec_certs.sample.certificate.References",
-                    "directly_referenced_by": null,
-                    "directly_referencing": null,
-                    "indirectly_referenced_by": null,
-                    "indirectly_referencing": null
-                },
-                "scheme_data": null
-            }
+        "st": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
         },
-        {
-            "_type": "sec_certs.sample.cc.CCCertificate",
-            "dgst": "8a5e6bcda602920c",
-            "status": "active",
-            "category": "Boundary Protection Devices and Systems",
-            "name": "Fortinet FortiGate w/ FortiOS v5.6.7",
-            "manufacturer": "Fortinet, Inc.",
-            "scheme": "CA",
-            "security_level": {
-                "_type": "Set",
-                "elements": []
-            },
-            "not_valid_before": "2019-05-22",
-            "not_valid_after": "2024-05-24",
-            "report_link": "https://www.commoncriteriaportal.org/files/epfiles/383-4-450%20CR%20v1.0a.pdf",
-            "st_link": "https://www.commoncriteriaportal.org/files/epfiles/383-4-450%20ST%20v1.3A.pdf",
-            "cert_link": "https://www.commoncriteriaportal.org/files/epfiles/383-4-450%20CT%20v1.0a.pdf",
-            "manufacturer_web": "https://www.fortinet.com/",
-            "protection_profiles": {
-                "_type": "Set",
-                "elements": [
-                    {
-                        "_type": "sec_certs.sample.protection_profile.ProtectionProfile",
-                        "pp_name": "collaborative Protection Profile for Stateful Traffic Filter Firewalls v2.0 + Errata 20180314",
-                        "pp_eal": null,
-                        "pp_link": "https://www.commoncriteriaportal.org/files/ppfiles/CPP_FW_V2.0E.pdf",
-                        "pp_ids": null
-                    }
-                ]
-            },
-            "maintenance_updates": {
-                "_type": "Set",
-                "elements": [
-                    {
-                        "_type": "sec_certs.sample.cc.CCCertificate.MaintenanceReport",
-                        "maintenance_date": "2019-08-26",
-                        "maintenance_title": "Fortinet FortiGate w/ FortiOS v5.6.7 Build 6022",
-                        "maintenance_report_link": "https://www.commoncriteriaportal.org/files/epfiles/383-7-159%20MR%20v1.0e.pdf",
-                        "maintenance_st_link": "https://www.commoncriteriaportal.org/files/epfiles/383-7-159%20ST%20v1.4%20CCRA.pdf"
-                    }
-                ]
-            },
-            "state": {
-                "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
-                "report": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                },
-                "st": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                },
-                "cert": {
-                    "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
-                    "download_ok": false,
-                    "convert_garbage": false,
-                    "convert_ok": false,
-                    "extract_ok": false,
-                    "pdf_hash": null,
-                    "txt_hash": null
-                }
-            },
-            "pdf_data": {
-                "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
-                "report_metadata": null,
-                "st_metadata": null,
-                "cert_metadata": null,
-                "report_frontpage": null,
-                "st_frontpage": null,
-                "cert_frontpage": null,
-                "report_keywords": null,
-                "st_keywords": null,
-                "cert_keywords": null,
-                "report_filename": null,
-                "st_filename": null,
-                "cert_filename": null
-            },
-            "heuristics": {
-                "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
-                "extracted_versions": null,
-                "cpe_matches": null,
-                "verified_cpe_matches": null,
-                "related_cves": null,
-                "cert_lab": null,
-                "cert_id": null,
-                "annotated_references": null,
-                "extracted_sars": null,
-                "direct_transitive_cves": null,
-                "indirect_transitive_cves": null,
-                "report_references": {
-                    "_type": "sec_certs.sample.certificate.References",
-                    "directly_referenced_by": null,
-                    "directly_referencing": null,
-                    "indirectly_referenced_by": null,
-                    "indirectly_referencing": null
-                },
-                "st_references": {
-                    "_type": "sec_certs.sample.certificate.References",
-                    "directly_referenced_by": null,
-                    "directly_referencing": null,
-                    "indirectly_referenced_by": null,
-                    "indirectly_referencing": null
-                },
-                "scheme_data": null
-            }
+        "cert": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
         }
-    ]
+      },
+      "pdf_data": {
+        "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
+        "report_metadata": null,
+        "st_metadata": null,
+        "cert_metadata": null,
+        "report_frontpage": null,
+        "st_frontpage": null,
+        "cert_frontpage": null,
+        "report_keywords": null,
+        "st_keywords": null,
+        "cert_keywords": null,
+        "report_filename": null,
+        "st_filename": null,
+        "cert_filename": null
+      },
+      "heuristics": {
+        "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
+        "extracted_versions": null,
+        "cpe_matches": null,
+        "verified_cpe_matches": null,
+        "related_cves": null,
+        "cert_lab": null,
+        "cert_id": null,
+        "annotated_references": null,
+        "extracted_sars": null,
+        "direct_transitive_cves": null,
+        "indirect_transitive_cves": null,
+        "report_references": {
+          "_type": "sec_certs.sample.certificate.References",
+          "directly_referenced_by": null,
+          "directly_referencing": null,
+          "indirectly_referenced_by": null,
+          "indirectly_referencing": null
+        },
+        "st_references": {
+          "_type": "sec_certs.sample.certificate.References",
+          "directly_referenced_by": null,
+          "directly_referencing": null,
+          "indirectly_referenced_by": null,
+          "indirectly_referencing": null
+        },
+        "scheme_data": null
+      }
+    },
+    {
+      "_type": "sec_certs.sample.cc.CCCertificate",
+      "dgst": "8a5e6bcda602920c",
+      "status": "active",
+      "category": "Boundary Protection Devices and Systems",
+      "name": "Fortinet FortiGate w/ FortiOS v5.6.7",
+      "manufacturer": "Fortinet, Inc.",
+      "scheme": "CA",
+      "security_level": {
+        "_type": "Set",
+        "elements": []
+      },
+      "not_valid_before": "2019-05-22",
+      "not_valid_after": "2024-05-24",
+      "report_link": "https://www.commoncriteriaportal.org/files/epfiles/383-4-450%20CR%20v1.0a.pdf",
+      "st_link": "https://www.commoncriteriaportal.org/files/epfiles/383-4-450%20ST%20v1.3A.pdf",
+      "cert_link": "https://www.commoncriteriaportal.org/files/epfiles/383-4-450%20CT%20v1.0a.pdf",
+      "manufacturer_web": "https://www.fortinet.com/",
+      "protection_profiles": {
+        "_type": "Set",
+        "elements": [
+          {
+            "_type": "sec_certs.sample.protection_profile.ProtectionProfile",
+            "pp_name": "collaborative Protection Profile for Stateful Traffic Filter Firewalls v2.0 + Errata 20180314",
+            "pp_eal": null,
+            "pp_link": "https://www.commoncriteriaportal.org/files/ppfiles/CPP_FW_V2.0E.pdf",
+            "pp_ids": null
+          }
+        ]
+      },
+      "maintenance_updates": {
+        "_type": "Set",
+        "elements": [
+          {
+            "_type": "sec_certs.sample.cc.CCCertificate.MaintenanceReport",
+            "maintenance_date": "2019-08-26",
+            "maintenance_title": "Fortinet FortiGate w/ FortiOS v5.6.7 Build 6022",
+            "maintenance_report_link": "https://www.commoncriteriaportal.org/files/epfiles/383-7-159%20MR%20v1.0e.pdf",
+            "maintenance_st_link": "https://www.commoncriteriaportal.org/files/epfiles/383-7-159%20ST%20v1.4%20CCRA.pdf"
+          }
+        ]
+      },
+      "state": {
+        "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
+        "report": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
+        },
+        "st": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
+        },
+        "cert": {
+          "_type": "sec_certs.sample.cc.CCCertificate.DocumentState",
+          "download_ok": false,
+          "convert_garbage": false,
+          "convert_ok": false,
+          "extract_ok": false,
+          "pdf_hash": null,
+          "txt_hash": null
+        }
+      },
+      "pdf_data": {
+        "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
+        "report_metadata": null,
+        "st_metadata": null,
+        "cert_metadata": null,
+        "report_frontpage": null,
+        "st_frontpage": null,
+        "cert_frontpage": null,
+        "report_keywords": null,
+        "st_keywords": null,
+        "cert_keywords": null,
+        "report_filename": null,
+        "st_filename": null,
+        "cert_filename": null
+      },
+      "heuristics": {
+        "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
+        "extracted_versions": null,
+        "cpe_matches": null,
+        "verified_cpe_matches": null,
+        "related_cves": null,
+        "cert_lab": null,
+        "cert_id": null,
+        "annotated_references": null,
+        "extracted_sars": null,
+        "direct_transitive_cves": null,
+        "indirect_transitive_cves": null,
+        "report_references": {
+          "_type": "sec_certs.sample.certificate.References",
+          "directly_referenced_by": null,
+          "directly_referencing": null,
+          "indirectly_referenced_by": null,
+          "indirectly_referencing": null
+        },
+        "st_references": {
+          "_type": "sec_certs.sample.certificate.References",
+          "directly_referenced_by": null,
+          "directly_referencing": null,
+          "indirectly_referenced_by": null,
+          "indirectly_referencing": null
+        },
+        "scheme_data": null
+      }
+    }
+  ]
 }
-- 
cgit v1.3.1


From 4592f97f15104968c1c49b9a04fe74f799b4039b Mon Sep 17 00:00:00 2001
From: J08nY
Date: Wed, 14 Feb 2024 13:26:56 +0100
Subject: Cleanup PdfData attributes.

---
 src/sec_certs/sample/cc.py                         | 103 +++++----------------
 src/sec_certs/utils/extract.py                     |   9 ++
 tests/data/cc/analysis/cc_full_dataset.json        |  13 +--
 tests/data/cc/analysis/reference_dataset.json      |  39 ++------
 .../analysis/transitive_vulnerability_dataset.json |  39 ++------
 5 files changed, 46 insertions(+), 157 deletions(-)

diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index bb4b0c7c..28099f58 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -27,15 +27,7 @@ from sec_certs.sample.sar import SAR
 from sec_certs.serialization.json import ComplexSerializableType
 from sec_certs.serialization.pandas import PandasSerializableType
 from sec_certs.utils import helpers
-from sec_certs.utils.extract import normalize_match_string
-
-HEADERS = {
-    "anssi": sec_certs.utils.extract.search_only_headers_anssi,
-    "bsi": sec_certs.utils.extract.search_only_headers_bsi,
-    "nscib": sec_certs.utils.extract.search_only_headers_nscib,
-    "niap": sec_certs.utils.extract.search_only_headers_niap,
-    "canada": sec_certs.utils.extract.search_only_headers_canada,
-}
+from sec_certs.utils.extract import normalize_match_string, scheme_frontpage_functions
 
 
 class CCCertificate(
@@ -181,8 +173,12 @@ class CCCertificate(
         st_metadata: dict[str, Any] | None = field(default=None)
         cert_metadata: dict[str, Any] | None = field(default=None)
         report_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
-        st_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
-        cert_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
+        st_frontpage: dict[str, dict[str, Any]] | None = field(
+            default=None
+        )  # TODO: Unused, we have no frontpage matching for targets
+        cert_frontpage: dict[str, dict[str, Any]] | None = field(
+            default=None
+        )  # TODO: Unused, we have no frontpage matching for certs
         report_keywords: dict[str, Any] | None = field(default=None)
         st_keywords: dict[str, Any] | None = field(default=None)
         cert_keywords: dict[str, Any] | None = field(default=None)
@@ -193,87 +189,34 @@ class CCCertificate(
         def __bool__(self) -> bool:
             return any(x is not None for x in vars(self))
 
-        @property
-        def bsi_data(self) -> dict[str, Any] | None:
-            """
-            Returns frontpage data related to BSI-provided information
-            """
-            return self.report_frontpage.get("bsi", None) if self.report_frontpage else None
-
-        @property
-        def niap_data(self) -> dict[str, Any] | None:
-            """
-            Returns frontpage data related to niap-provided information
-            """
-            return self.report_frontpage.get("niap", None) if self.report_frontpage else None
-
-        @property
-        def nscib_data(self) -> dict[str, Any] | None:
-            """
-            Returns frontpage data related to nscib-provided information
-            """
-            return self.report_frontpage.get("nscib", None) if self.report_frontpage else None
-
-        @property
-        def canada_data(self) -> dict[str, Any] | None:
-            """
-            Returns frontpage data related to canada-provided information
-            """
-            return self.report_frontpage.get("canada", None) if self.report_frontpage else None
-
-        @property
-        def anssi_data(self) -> dict[str, Any] | None:
-            """
-            Returns frontpage data related to ANSSI-provided information
-            """
-            return self.report_frontpage.get("anssi", None) if self.report_frontpage else None
-
         @property
         def cert_lab(self) -> list[str] | None:
             """
             Returns labs for which certificate data was parsed.
             """
+            if not self.report_frontpage:
+                return None
             labs = [
                 data["cert_lab"].split(" ")[0].upper()
-                for data in [self.bsi_data, self.anssi_data, self.niap_data, self.nscib_data, self.canada_data]
+                for scheme, data in self.report_frontpage.items()
                 if data and "cert_lab" in data
             ]
             return labs if labs else None
 
-        @property
-        def bsi_cert_id(self) -> str | None:
-            return self.bsi_data.get("cert_id", None) if self.bsi_data else None
-
-        @property
-        def niap_cert_id(self) -> str | None:
-            return self.niap_data.get("cert_id", None) if self.niap_data else None
-
-        @property
-        def nscib_cert_id(self) -> str | None:
-            return self.nscib_data.get("cert_id", None) if self.nscib_data else None
-
-        @property
-        def canada_cert_id(self) -> str | None:
-            return self.canada_data.get("cert_id", None) if self.canada_data else None
-
-        @property
-        def anssi_cert_id(self) -> str | None:
-            return self.anssi_data.get("cert_id", None) if self.anssi_data else None
-
         def frontpage_cert_id(self, scheme: str) -> dict[str, float]:
             """
             Get cert_id candidate from the frontpage of the report.
             """
-            scheme_map = {
-                "DE": self.bsi_cert_id,
-                "US": self.niap_cert_id,
-                "NL": self.nscib_cert_id,
-                "CA": self.canada_cert_id,
-                "FR": self.anssi_cert_id,
-            }
-            if scheme in scheme_map and (candidate := scheme_map[scheme]):
-                return {candidate: 1.0}
-            return {}
+            if not self.report_frontpage:
+                return {}
+            data = self.report_frontpage.get(scheme)
+            if not data:
+                return {}
+            cert_id = data.get("cert_id")
+            if not cert_id:
+                return {}
+            else:
+                return {cert_id: 1.0}
 
         def filename_cert_id(self, scheme: str) -> dict[str, float]:
             """
@@ -982,9 +925,9 @@ class CCCertificate(
         """
         cert.pdf_data.report_frontpage = {}
 
-        for header_type, associated_header_func in HEADERS.items():
-            response, cert.pdf_data.report_frontpage[header_type] = associated_header_func(cert.state.report.txt_path)
-
+        if cert.scheme in scheme_frontpage_functions:
+            header_func = scheme_frontpage_functions[cert.scheme]
+            response, cert.pdf_data.report_frontpage[cert.scheme] = header_func(cert.state.report.txt_path)
             if response != constants.RETURNCODE_OK:
                 cert.state.report.extract_ok = False
         return cert
diff --git a/src/sec_certs/utils/extract.py b/src/sec_certs/utils/extract.py
index e6312b8d..669cbcfc 100644
--- a/src/sec_certs/utils/extract.py
+++ b/src/sec_certs/utils/extract.py
@@ -818,3 +818,12 @@ def get_sums_for_rules_subset(dct: dict | None, path: str) -> dict[str, float]:
     cc_rules_subset_to_search = rules_get_subset(path)
     paths_to_search = extract_key_paths(cc_rules_subset_to_search, path)
     return {x: get_sum_of_values_from_dict_path(dct, x, np.nan) for x in paths_to_search}
+
+
+scheme_frontpage_functions = {
+    "FR": search_only_headers_anssi,
+    "DE": search_only_headers_bsi,
+    "NL": search_only_headers_nscib,
+    "US": search_only_headers_niap,
+    "CA": search_only_headers_canada,
+}
diff --git a/tests/data/cc/analysis/cc_full_dataset.json b/tests/data/cc/analysis/cc_full_dataset.json
index 4baf4758..64ef38e4 100644
--- a/tests/data/cc/analysis/cc_full_dataset.json
+++ b/tests/data/cc/analysis/cc_full_dataset.json
@@ -130,8 +130,7 @@
                     }
                 },
                 "report_frontpage": {
-                    "anssi": {},
-                    "bsi": {
+                    "DE": {
                         "match_rules": [
                             "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)"
                         ],
@@ -142,17 +141,9 @@
                         "ref_protection_profiles": "None",
                         "cc_version": "Product specific Security Target Common Criteria Part 2 conformant",
                         "cc_security_level": "Common Criteria Part 3 conformant EAL 3 augmented by ALC_FLR.1 SOGIS Recognition Agreement"
-                    },
-                    "nscib": {},
-                    "niap": {},
-                    "canada": {}
+                    }
                 },
                 "st_frontpage": {
-                    "anssi": {},
-                    "bsi": {},
-                    "nscib": {},
-                    "niap": {},
-                    "canada": {}
                 },
                 "report_keywords": {
                     "cc_cert_id": {
diff --git a/tests/data/cc/analysis/reference_dataset.json b/tests/data/cc/analysis/reference_dataset.json
index b0ef6683..38fb7fd0 100644
--- a/tests/data/cc/analysis/reference_dataset.json
+++ b/tests/data/cc/analysis/reference_dataset.json
@@ -101,8 +101,7 @@
           "/Title": "Security Target The Océ Digital Access Controller (DAC) R10.1.5, as used in the Océ VarioPrint 1055, 1055 BC, 1055 DP, 1065, 1075, 2062, 2075, 2075 DP printer/copier/scanner products"
         },
         "report_frontpage": {
-          "anssi": {},
-          "bsi": {
+          "DE": {
             "match_rules": [
               "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)"
             ],
@@ -110,17 +109,9 @@
             "cert_item": "Océ Digital Access Controller (DAC) R10.1.5 for use in the Océ VarioPrint 1055, 1055 BC, 1055 DP, 1065, 1075, 2062, 2075, 2075 DP printer/copier/scanner products",
             "developer": "Océ Technologies B.V",
             "cert_lab": "BSI"
-          },
-          "nscib": {},
-          "niap": {},
-          "canada": {}
+          }
         },
         "st_frontpage": {
-          "anssi": {},
-          "bsi": {},
-          "nscib": {},
-          "niap": {},
-          "canada": {}
         },
         "report_keywords": {
           "cc_cert_id": {
@@ -678,8 +669,7 @@
           "/Title": "Microsoft Word - Oce Venlo DAC Security Target 2.4.doc"
         },
         "report_frontpage": {
-          "anssi": {},
-          "bsi": {
+          "DE": {
             "match_rules": [
               "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)"
             ],
@@ -687,17 +677,9 @@
             "cert_item": "Océ Digital Access Controller (DAC) R9.1.6",
             "developer": "Océ Technologies B.V",
             "cert_lab": "BSI"
-          },
-          "nscib": {},
-          "niap": {},
-          "canada": {}
+          }
         },
         "st_frontpage": {
-          "anssi": {},
-          "bsi": {},
-          "nscib": {},
-          "niap": {},
-          "canada": {}
         },
         "report_keywords": {
           "cc_cert_id": {
@@ -1333,8 +1315,7 @@
           "/Title": "Microsoft Word - Oce Venlo DAC Security Target 1.9.doc"
         },
         "report_frontpage": {
-          "anssi": {},
-          "bsi": {
+          "DE": {
             "match_rules": [
               "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)"
             ],
@@ -1342,17 +1323,9 @@
             "cert_item": "Océ Digital Access Controller (DAC) R 8.1.10",
             "developer": "Océ Technologies B.V",
             "cert_lab": "BSI"
-          },
-          "nscib": {},
-          "niap": {},
-          "canada": {}
+          }
         },
         "st_frontpage": {
-          "anssi": {},
-          "bsi": {},
-          "nscib": {},
-          "niap": {},
-          "canada": {}
         },
         "report_keywords": {
           "cc_cert_id": {
diff --git a/tests/data/cc/analysis/transitive_vulnerability_dataset.json b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
index 21391a12..abd4c7a3 100644
--- a/tests/data/cc/analysis/transitive_vulnerability_dataset.json
+++ b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
@@ -108,8 +108,7 @@
           "/CreationDate": "D:20140828154014+02'00'"
         },
         "report_frontpage": {
-          "anssi": {},
-          "bsi": {
+          "DE": {
             "match_rules": [
               "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)"
             ],
@@ -117,17 +116,9 @@
             "cert_item": "IBM z/OS Version 2 Release 1",
             "developer": "IBM Corporation",
             "cert_lab": "BSI"
-          },
-          "nscib": {},
-          "niap": {},
-          "canada": {}
+          }
         },
         "st_frontpage": {
-          "anssi": {},
-          "bsi": {},
-          "nscib": {},
-          "niap": {},
-          "canada": {}
         },
         "report_keywords": {
           "cc_cert_id": {
@@ -1432,8 +1423,7 @@
           "/CreationDate": "D:20150508083715+02'00'"
         },
         "report_frontpage": {
-          "anssi": {},
-          "bsi": {
+          "DE": {
             "match_rules": [
               "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)"
             ],
@@ -1441,17 +1431,9 @@
             "cert_item": "RACF Element of z/OS Version 2, Release 1",
             "developer": "IBM Corporation",
             "cert_lab": "BSI"
-          },
-          "nscib": {},
-          "niap": {},
-          "canada": {}
+          }
         },
         "st_frontpage": {
-          "anssi": {},
-          "bsi": {},
-          "nscib": {},
-          "niap": {},
-          "canada": {}
         },
         "report_keywords": {
           "cc_cert_id": {
@@ -2399,8 +2381,7 @@
           "/CreationDate": "D:20170503171742+02'00'"
         },
         "report_frontpage": {
-          "anssi": {},
-          "bsi": {
+          "DE": {
             "match_rules": [
               "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)"
             ],
@@ -2411,17 +2392,9 @@
             "ref_protection_profiles": "Operating System Protection Profile, Version 2.0, 01 June 2010, BSI-CC-PP-0067-2010, OSPP Extended Packages: Extended Identification and Authentication and Labeled Security, both Version 2.0, 28 May 2010",
             "cc_version": "PP conformant Common Criteria Part 2 extended",
             "cc_security_level": "Common Criteria Part 3 conformant EAL 4 augmented by ALC_FLR.3"
-          },
-          "nscib": {},
-          "niap": {},
-          "canada": {}
+          }
         },
         "st_frontpage": {
-          "anssi": {},
-          "bsi": {},
-          "nscib": {},
-          "niap": {},
-          "canada": {}
         },
         "report_keywords": {
           "cc_cert_id": {
-- 
cgit v1.3.1


From 042c527698fe41dd306b64437f44b45760c2d074 Mon Sep 17 00:00:00 2001
From: J08nY
Date: Wed, 14 Feb 2024 16:37:45 +0100
Subject: Fix typo.

---
 src/sec_certs/sample/sar.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sec_certs/sample/sar.py b/src/sec_certs/sample/sar.py
index e3d7bd2b..ea4a6e14 100644
--- a/src/sec_certs/sample/sar.py
+++ b/src/sec_certs/sample/sar.py
@@ -15,7 +15,7 @@ SAR_CLASS_MAPPING = {
     "ALC": "Life-cycle support",
     "ATE": "Tests",
     "AVA": "Vulnerability assessment",
-    "ACO": "Comoposition",
+    "ACO": "Composition",
 }
 
 SAR_CLASSES = set(SAR_CLASS_MAPPING)
-- 
cgit v1.3.1


From ded1c20025f06eace26f0b0dd15f4a89fbac51a8 Mon Sep 17 00:00:00 2001
From: dependabot[bot]
Date: Sat, 17 Feb 2024 00:58:07 +0000
Subject: chore(deps): bump cryptography from 42.0.0 to 42.0.2 in /requirements

Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.0 to 42.0.2.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/42.0.0...42.0.2)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>---
 requirements/all_requirements.txt  | 2 +-
 requirements/dev_requirements.txt  | 2 +-
 requirements/nlp_requirements.txt  | 2 +-
 requirements/requirements.txt      | 2 +-
 requirements/test_requirements.txt | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index ae9e742c..72cffbc0 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -94,7 +94,7 @@ coverage[toml]==7.3.2
     # via
     #   pytest-cov
     #   sec-certs (./../pyproject.toml)
-cryptography==42.0.0
+cryptography==42.0.2
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index f340bbd9..deadf93b 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -71,7 +71,7 @@ coverage[toml]==7.3.2
     # via
     #   coverage
     #   pytest-cov
-cryptography==42.0.0
+cryptography==42.0.2
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt
index f06f83c8..012a263e 100644
--- a/requirements/nlp_requirements.txt
+++ b/requirements/nlp_requirements.txt
@@ -73,7 +73,7 @@ contourpy==1.2.0
     # via
     #   bokeh
     #   matplotlib
-cryptography==42.0.0
+cryptography==42.0.2
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 3ecf9f39..1b17a94c 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -41,7 +41,7 @@ confection==0.1.3
     #   weasel
 contourpy==1.2.0
     # via matplotlib
-cryptography==42.0.0
+cryptography==42.0.2
     # via pypdf
 cycler==0.12.1
     # via matplotlib
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 3a2a5198..f94e9328 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -45,7 +45,7 @@ coverage[toml]==7.3.2
     # via
     #   pytest-cov
     #   sec-certs (./../pyproject.toml)
-cryptography==42.0.0
+cryptography==42.0.2
     # via pypdf
 cycler==0.12.1
     # via matplotlib
-- 
cgit v1.3.1