merge: merged main into feature branch

8 files changed, 140 insertions, 68 deletions

diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 603c29f2..9f73aefa 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -17,13 +17,12 @@ from bs4 import Tag
 import sec_certs.utils.extract
 import sec_certs.utils.pdf
 import sec_certs.utils.sanitization
-from sec_certs import constants as constants
+from sec_certs import constants
 from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, cc_rules, rules, security_level_csv_scan
 from sec_certs.sample.cc_certificate_id import canonicalize
-from sec_certs.sample.certificate import Certificate
+from sec_certs.sample.certificate import Certificate, References, logger
 from sec_certs.sample.certificate import Heuristics as BaseHeuristics
 from sec_certs.sample.certificate import PdfData as BasePdfData
-from sec_certs.sample.certificate import References, logger
 from sec_certs.sample.protection_profile import ProtectionProfile
 from sec_certs.sample.sar import SAR
 from sec_certs.serialization.json import ComplexSerializableType
@@ -117,10 +116,10 @@ class CCCertificate(
         st_txt_hash: str | None
         report_txt_hash: str | None
 
-        st_pdf_path: Path
-        report_pdf_path: Path
-        st_txt_path: Path
-        report_txt_path: Path
+        _st_pdf_path: Path | None = None
+        _report_pdf_path: Path | None = None
+        _st_txt_path: Path | None = None
+        _report_txt_path: Path | None = None
 
         def __init__(
             self,
@@ -152,6 +151,46 @@ class CCCertificate(
             self.report_txt_hash = report_txt_hash
 
         @property
+        def st_pdf_path(self) -> Path:
+            if not self._st_pdf_path:
+                raise ValueError(f"st_pdf_path not set on {type(self)}")
+            return self._st_pdf_path
+
+        @st_pdf_path.setter
+        def st_pdf_path(self, pth: str | Path | None) -> None:
+            self._st_pdf_path = Path(pth) if pth else None
+
+        @property
+        def report_pdf_path(self) -> Path:
+            if not self._report_pdf_path:
+                raise ValueError(f"report_pdf_path not set on {type(self)}")
+            return self._report_pdf_path
+
+        @report_pdf_path.setter
+        def report_pdf_path(self, pth: str | Path | None) -> None:
+            self._report_pdf_path = Path(pth) if pth else None
+
+        @property
+        def st_txt_path(self) -> Path:
+            if not self._st_txt_path:
+                raise ValueError(f"st_txt_path not set on {type(self)}")
+            return self._st_txt_path
+
+        @st_txt_path.setter
+        def st_txt_path(self, pth: str | Path | None) -> None:
+            self._st_txt_path = Path(pth) if pth else None
+
+        @property
+        def report_txt_path(self) -> Path:
+            if not self._report_txt_path:
+                raise ValueError(f"report_txt_path not set on {type(self)}")
+            return self._report_txt_path
+
+        @report_txt_path.setter
+        def report_txt_path(self, pth: str | Path | None) -> None:
+            self._report_txt_path = Path(pth) if pth else None
+
+        @property
         def serialized_attributes(self) -> list[str]:
             return [
                 "st_download_ok",
@@ -472,9 +511,9 @@ class CCCertificate(
         self.manufacturer_web = sec_certs.utils.sanitization.sanitize_link(manufacturer_web)
         self.protection_profiles = protection_profiles
         self.maintenance_updates = maintenance_updates
-        self.state = self.InternalState() if not state else state
-        self.pdf_data = self.PdfData() if not pdf_data else pdf_data
-        self.heuristics: CCCertificate.Heuristics = self.Heuristics() if not heuristics else heuristics
+        self.state = state if state else self.InternalState()
+        self.pdf_data = pdf_data if pdf_data else self.PdfData()
+        self.heuristics: CCCertificate.Heuristics = heuristics if heuristics else self.Heuristics()
 
     @property
     def dgst(self) -> str:
@@ -507,7 +546,7 @@ class CCCertificate(
         Computes actual SARs. First, SARs implied by EAL are computed. Then, these are augmented with heuristically extracted SARs
         :return Optional[Set[SAR]]: Set of actual SARs of a certificate, None if empty
         """
-        sars = dict()
+        sars = {}
         if self.eal:
             sars = {x[0]: SAR(x[0], x[1]) for x in SARS_IMPLIED_FROM_EAL[self.eal[:4]]}
 
@@ -755,13 +794,13 @@ class CCCertificate(
         :param Optional[Union[str, Path]] report_txt_dir: Directory where txt reports shall be stored
         :param Optional[Union[str, Path]] st_txt_dir: Directory where txt security targets shall be stored
         """
-        if report_pdf_dir is not None:
+        if report_pdf_dir:
             self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf")
-        if st_pdf_dir is not None:
+        if st_pdf_dir:
             self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf")
-        if report_txt_dir is not None:
+        if report_txt_dir:
             self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + ".txt")
-        if st_txt_dir is not None:
+        if st_txt_dir:
             self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + ".txt")
 
     @staticmethod
@@ -795,11 +834,10 @@ class CCCertificate(
         :param CCCertificate cert: cert to download the pdf security target for
         :return CCCertificate: returns the modified certificate with updated state
         """
-        exit_code: str | int
-        if not cert.st_link:
-            exit_code = "No link"
-        else:
-            exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path)
+        exit_code: str | int = (
+            helpers.download_file(cert.st_link, cert.state.st_pdf_path) if cert.st_link else "No link"
+        )
+
         if exit_code != requests.codes.ok:
             error_msg = f"failed to download ST from {cert.st_link}, code: {exit_code}"
             logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
diff --git a/src/sec_certs/sample/cc_maintenance_update.py b/src/sec_certs/sample/cc_maintenance_update.py
index d78359fc..4273b25e 100644
--- a/src/sec_certs/sample/cc_maintenance_update.py
+++ b/src/sec_certs/sample/cc_maintenance_update.py
@@ -4,9 +4,9 @@ import logging
 from datetime import date
 from typing import ClassVar
 
-import sec_certs.utils.helpers as helpers
 from sec_certs.sample.cc import CCCertificate
 from sec_certs.serialization.json import ComplexSerializableType
+from sec_certs.utils import helpers
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/sec_certs/sample/certificate.py b/src/sec_certs/sample/certificate.py
index bb49c0df..5dab9cef 100644
--- a/src/sec_certs/sample/certificate.py
+++ b/src/sec_certs/sample/certificate.py
@@ -25,6 +25,9 @@ class References(ComplexSerializableType):
     directly_referencing: set[str] | None = field(default=None)
     indirectly_referencing: set[str] | None = field(default=None)
 
+    def __bool__(self):
+        return any(getattr(self, x) for x in vars(self))
+
 
 class Heuristics:
     cpe_matches: set[str] | None
diff --git a/src/sec_certs/sample/fips.py b/src/sec_certs/sample/fips.py
index aec791d4..b3ff8ba6 100644
--- a/src/sec_certs/sample/fips.py
+++ b/src/sec_certs/sample/fips.py
@@ -14,21 +14,16 @@ import requests
 from bs4 import BeautifulSoup, Tag
 from tabula import read_pdf
 
-import sec_certs.constants as constants
-import sec_certs.utils.extract
-import sec_certs.utils.helpers as helpers
-import sec_certs.utils.pdf
-import sec_certs.utils.pdf as pdf
-import sec_certs.utils.tables as tables
+from sec_certs import constants
 from sec_certs.cert_rules import FIPS_ALGS_IN_TABLE, fips_rules
 from sec_certs.config.configuration import config
-from sec_certs.sample.certificate import Certificate
+from sec_certs.sample.certificate import Certificate, References, logger
 from sec_certs.sample.certificate import Heuristics as BaseHeuristics
 from sec_certs.sample.certificate import PdfData as BasePdfData
-from sec_certs.sample.certificate import References, logger
 from sec_certs.sample.cpe import CPE
 from sec_certs.serialization.json import ComplexSerializableType
 from sec_certs.serialization.pandas import PandasSerializableType
+from sec_certs.utils import extract, helpers, pdf, tables
 from sec_certs.utils.helpers import fips_dgst
 
 
@@ -69,7 +64,7 @@ class FIPSHTMLParser:
             [x.find("div", class_="col-md-3") for x in entries], [x.find("div", class_="col-md-9") for x in entries]
         )
         entries = [(FIPSHTMLParser.normalize_string(key.text), entry) for key, entry in entries]
-        entries = [parse_single_detail_entry(*x) for x in entries if x[0] in DETAILS_KEY_NORMALIZATION_DICT.keys()]
+        entries = [parse_single_detail_entry(*x) for x in entries if x[0] in DETAILS_KEY_NORMALIZATION_DICT]
         entries = {x: y for x, y in entries}
 
         if "caveat" in entries:
@@ -130,11 +125,22 @@ class FIPSHTMLParser:
 
     @staticmethod
     def parse_algorithms(algorithms_div: Tag) -> dict[str, set[str]]:
-        rows = algorithms_div.find("tbody").find_all("tr")
-        dct: dict[str, set[str]] = dict()
-        for row in rows:
-            cells = row.find_all("td")
-            dct[cells[0].text] = {m.group() for m in re.finditer(FIPS_ALGS_IN_TABLE, cells[1].text)}
+        dct: dict[str, set[str]] = {}
+        table = algorithms_div.find("tbody")
+        # Two types of organization on the CMVP website:
+        #  - One is a table with algo references in text
+        #  - Other is just divs for rows, one per algo
+        if table:
+            rows = table.find_all("tr")
+            for row in rows:
+                cells = row.find_all("td")
+                dct[str(cells[0].text)] = {m.group() for m in re.finditer(FIPS_ALGS_IN_TABLE, cells[1].text)}
+        else:
+            rows = algorithms_div.find_all("div", class_="col-md-12")
+            for row in rows:
+                dct[str(row.find("div", class_="col-md-3").text)] = {
+                    str(row.find("div", class_="col-md-4").text).strip()
+                }
         return dct
 
     @staticmethod
@@ -144,7 +150,7 @@ class FIPSHTMLParser:
     @staticmethod
     def parse_tested_configurations(tested_configurations: Tag) -> list[str] | None:
         configurations = [y.text for y in tested_configurations.find_all("li")]
-        return configurations if not configurations == ["N/A"] else None
+        return None if configurations == ["N/A"] else configurations
 
     @staticmethod
     def normalize_embodiment(embodiment_element: Tag) -> str:
@@ -248,9 +254,9 @@ class FIPSCertificate(
         policy_pdf_hash: str | None
         policy_txt_hash: str | None
 
-        policy_pdf_path: Path
-        policy_txt_path: Path
-        module_html_path: Path
+        _policy_pdf_path: Path | None = None
+        _policy_txt_path: Path | None = None
+        _module_html_path: Path | None = None
 
         def __init__(
             self,
@@ -273,6 +279,36 @@ class FIPSCertificate(
             self.policy_txt_hash = policy_txt_hash
 
         @property
+        def policy_pdf_path(self) -> Path:
+            if not self._policy_pdf_path:
+                raise ValueError(f"policy_pdf_path not set on {type(self)}")
+            return self._policy_pdf_path
+
+        @policy_pdf_path.setter
+        def policy_pdf_path(self, pth: str | Path | None) -> None:
+            self._policy_pdf_path = Path(pth) if pth else None
+
+        @property
+        def policy_txt_path(self) -> Path:
+            if not self._policy_txt_path:
+                raise ValueError(f"policy_txt_path not set on {type(self)}")
+            return self._policy_txt_path
+
+        @policy_txt_path.setter
+        def policy_txt_path(self, pth: str | Path | None) -> None:
+            self._policy_txt_path = Path(pth) if pth else None
+
+        @property
+        def module_html_path(self) -> Path:
+            if not self._module_html_path:
+                raise ValueError(f"module_html_path not set on {type(self)}")
+            return self._module_html_path
+
+        @module_html_path.setter
+        def module_html_path(self, pth: str | Path | None) -> None:
+            self._module_html_path = Path(pth) if pth else None
+
+        @property
         def serialized_attributes(self) -> list[str]:
             return [
                 "module_download_ok",
@@ -389,8 +425,8 @@ class FIPSCertificate(
         def certlike_algorithm_numbers(self) -> set[str]:
             """Returns numbers of certificates from keywords["fips_certlike"]["Certlike"]"""
             if self.keywords and "fips_certlike" in self.keywords:
-                fips_certlike = self.keywords["fips_certlike"].get("Certlike", dict())
-                matches = {re.search(r"#\s{0,1}\d{1,4}", x) for x in fips_certlike.keys()}
+                fips_certlike = self.keywords["fips_certlike"].get("Certlike", {})
+                matches = {re.search(r"#\s{0,1}\d{1,4}", x) for x in fips_certlike}
                 return {"".join([x for x in match.group() if x.isdigit()]) for match in matches if match}
             else:
                 return set()
@@ -463,7 +499,7 @@ class FIPSCertificate(
 
     def __init__(
         self,
-        cert_id: str,
+        cert_id: int,
         web_data: FIPSCertificate.WebData | None = None,
         pdf_data: FIPSCertificate.PdfData | None = None,
         heuristics: FIPSCertificate.Heuristics | None = None,
@@ -543,9 +579,7 @@ class FIPSCertificate(
         """
         Converts policy pdf -> txt
         """
-        ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(
-            cert.state.policy_pdf_path, cert.state.policy_txt_path
-        )
+        ocr_done, ok_result = pdf.convert_pdf_file(cert.state.policy_pdf_path, cert.state.policy_txt_path)
 
         # If OCR was done and the result was garbage
         cert.state.policy_convert_garbage = ocr_done
@@ -565,12 +599,12 @@ class FIPSCertificate(
         """
         Extract the PDF metadata from the security policy.
         """
-        _, metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.policy_pdf_path)
+        _, metadata = pdf.extract_pdf_metadata(cert.state.policy_pdf_path)
 
         if metadata:
             cert.pdf_data.policy_metadata = metadata
         else:
-            cert.pdf_data.policy_metadata = dict()
+            cert.pdf_data.policy_metadata = {}
             cert.state.policy_extract_ok = False
         return cert
 
@@ -579,7 +613,7 @@ class FIPSCertificate(
         """
         Extract keywords from policy document
         """
-        keywords = sec_certs.utils.extract.extract_keywords(cert.state.policy_txt_path, fips_rules)
+        keywords = extract.extract_keywords(cert.state.policy_txt_path, fips_rules)
         if not keywords:
             cert.state.policy_extract_ok = False
         else:
@@ -618,7 +652,7 @@ class FIPSCertificate(
         self.heuristics.module_prunned_references = self._prune_reference_ids_variable(html_module_ids)
 
         if self.pdf_data.keywords:
-            pdf_policy_ids = set(self.pdf_data.keywords["fips_cert_id"].get("Cert", dict()).keys())
+            pdf_policy_ids = set(self.pdf_data.keywords["fips_cert_id"].get("Cert", {}).keys())
             pdf_policy_ids = {"".join([y for y in x if y.isdigit()]) for x in pdf_policy_ids}
         else:
             pdf_policy_ids = set()
@@ -646,9 +680,7 @@ class FIPSCertificate(
             2. Consider only ids s.t. they don't appear in self.heuristics.algorithms
             3. Consider only ids s.t. they don't appear in self.pdf_data.keywords["fips_certlike"]["Certlike"]
         """
-        prunned = {x for x in attribute_to_prune if x != self.cert_id}
+        prunned = {x for x in attribute_to_prune if x != str(self.cert_id)}
         prunned = {x for x in prunned if int(x) > config.always_false_positive_fips_cert_id_threshold}
         prunned = {x for x in prunned if x not in self.heuristics.algorithm_numbers}
-        prunned = {x for x in prunned if x not in self.pdf_data.certlike_algorithm_numbers}
-
-        return prunned
+        return {x for x in prunned if x not in self.pdf_data.certlike_algorithm_numbers}
diff --git a/src/sec_certs/sample/fips_iut.py b/src/sec_certs/sample/fips_iut.py
index cb521ee4..f6010346 100644
--- a/src/sec_certs/sample/fips_iut.py
+++ b/src/sec_certs/sample/fips_iut.py
@@ -85,10 +85,11 @@ class IUTSnapshot(ComplexSerializableType):
 
         last_updated_elem = next(
             filter(
-                lambda e: isinstance(e, Tag) and e.name == "p",
+                lambda e: isinstance(e, Tag) and e.name == "p" and "Last Updated" in str(e.string),
                 soup.find(id="content").next_siblings,
             )
         )
+
         last_updated_text = str(last_updated_elem.string).strip()
         last_updated = datetime.strptime(last_updated_text, "Last Updated: %m/%d/%Y").date()
         table = tables[0].find("tbody")
@@ -100,7 +101,7 @@ class IUTSnapshot(ComplexSerializableType):
                 str(line[2].string),
                 datetime.strptime(str(line[3].string), "%m/%d/%Y").date(),
             )
-            for line in map(lambda tr: tr.find_all("td"), lines)
+            for line in (tr.find_all("td") for tr in lines)
         }
 
         # Parse footer
diff --git a/src/sec_certs/sample/fips_mip.py b/src/sec_certs/sample/fips_mip.py
index 6918d2aa..7e0ddff0 100644
--- a/src/sec_certs/sample/fips_mip.py
+++ b/src/sec_certs/sample/fips_mip.py
@@ -114,7 +114,7 @@ class MIPSnapshot(ComplexSerializableType):
             MIPEntry(
                 str(line[0].string), str(line[1].string), str(line[2].string), MIPStatus(str(line[3].string)), None
             )
-            for line in map(lambda tr: tr.find_all("td"), lines)
+            for line in (tr.find_all("td") for tr in lines)
         }
 
     @classmethod
@@ -128,14 +128,14 @@ class MIPSnapshot(ComplexSerializableType):
                 MIPStatus(str(line[3].string)),
                 None,
             )
-            for line in map(lambda tr: tr.find_all("td"), lines)
+            for line in (tr.find_all("td") for tr in lines)
         }
 
     @classmethod
     def _extract_entries_4(cls, lines):
         """Works now."""
         entries = set()
-        for line in map(lambda tr: tr.find_all("td"), lines):
+        for line in (tr.find_all("td") for tr in lines):
             module_name = str(line[0].string)
             vendor_name = str(" ".join(line[1].find_all(text=True, recursive=False)).strip())
             standard = str(line[2].string)
@@ -150,14 +150,12 @@ class MIPSnapshot(ComplexSerializableType):
     @classmethod
     def _extract_entries(cls, lines, snapshot_date):
         if snapshot_date <= datetime(2020, 10, 28):
-            entries = cls._extract_entries_1(lines)
-        elif snapshot_date <= datetime(2021, 4, 20):
-            entries = cls._extract_entries_2(lines)
-        elif snapshot_date <= datetime(2022, 3, 23):
-            entries = cls._extract_entries_3(lines)
-        else:
-            entries = cls._extract_entries_4(lines)
-        return entries
+            return cls._extract_entries_1(lines)
+        if snapshot_date <= datetime(2021, 4, 20):
+            return cls._extract_entries_2(lines)
+        if snapshot_date <= datetime(2022, 3, 23):
+            return cls._extract_entries_3(lines)
+        return cls._extract_entries_4(lines)
 
     @classmethod
     def from_page(cls, content: bytes, snapshot_date: datetime) -> MIPSnapshot:
diff --git a/src/sec_certs/sample/protection_profile.py b/src/sec_certs/sample/protection_profile.py
index b7c2ec34..4c26a1c7 100644
--- a/src/sec_certs/sample/protection_profile.py
+++ b/src/sec_certs/sample/protection_profile.py
@@ -5,8 +5,8 @@ import logging
 from dataclasses import dataclass
 from typing import Any
 
-import sec_certs.utils.sanitization as sanitization
 from sec_certs.serialization.json import ComplexSerializableType
+from sec_certs.utils import sanitization
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/sec_certs/sample/sar.py b/src/sec_certs/sample/sar.py
index 31359299..8f48f417 100644
--- a/src/sec_certs/sample/sar.py
+++ b/src/sec_certs/sample/sar.py
@@ -18,7 +18,7 @@ SAR_CLASS_MAPPING = {
     "ACO": "Comoposition",
 }
 
-SAR_CLASSES = {x for x in SAR_CLASS_MAPPING}
+SAR_CLASSES = set(SAR_CLASS_MAPPING)
 SAR_DICT_KEY = "cc_sar"