Merge pull request #478 from crocs-muni/fix/mutable-def-arg

Fix mutable default state.

21 files changed, 65 insertions, 91 deletions

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5b3353fa..46b7773c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -31,7 +31,7 @@ jobs:
           pip install -e .
           python -m spacy download en_core_web_sm
       - name: Run tests
-        run: pytest --cov=sec_certs tests
+        run: pytest
       - name: Code coverage upload
         uses: codecov/codecov-action@v4
         with:
diff --git a/pyproject.toml b/pyproject.toml
index a13f9e77..f27c9aa6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,6 +21,7 @@
     "Topic :: Security :: Cryptography",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
   ]
@@ -83,7 +84,7 @@
       "sphinx-copybutton",
       "ipython!=8.7.0",
     ]
-    test = ["pytest", "coverage", "pytest-cov"]
+    test = ["pytest", "coverage[toml]", "pytest-cov"]
     nlp = [
       "catboost",
       "optuna",
@@ -124,20 +125,18 @@
   [tool.ruff.lint.mccabe]
     max-complexity = 10
 
-  [tool.setuptools.package-data]
-    'sec_certs' = ["rules.yaml"]
-    'sec_certs.config' = ["settings.yaml", "settings-schema.json"]
-    'sec_certs.data' = [
-      "reference_annotations/split/*.json",
-      "reference_annotations/manual_annotations/final/*.csv",
-    ]
-
+[tool.setuptools.package-data]
+  'sec_certs' = ["rules.yaml"]
+  'sec_certs.config' = ["settings.yaml", "settings-schema.json"]
+  'sec_certs.data' = [
+    "reference_annotations/split/*.json",
+    "reference_annotations/manual_annotations/final/*.csv",
+  ]
 
 [tool.setuptools_scm]
   write_to = "src/sec_certs/_version.py"
   version_scheme = "no-guess-dev"
 
-
 [tool.mypy]
   plugins = ["numpy.typing.mypy_plugin"]
   ignore_missing_imports = true
@@ -145,4 +144,6 @@
 
 [tool.pytest.ini_options]
   markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]
-  addopts = "--cov sec_certs"
+
+[tool.coverage.run]
+  source = ["src"]
diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index fb90c142..dc1c52b4 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -91,33 +91,23 @@ class CCDataset(Dataset[CCCertificate], ComplexSerializableType):
 
     def __init__(
         self,
-        certs: dict[str, CCCertificate] = {},
+        certs: dict[str, CCCertificate] | None = None,
         root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH,
         name: str | None = None,
         description: str = "",
         state: Dataset.DatasetInternalState | None = None,
-        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] = {},
+        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] | None = None,
     ):
-        self.certs = certs
-        self.timestamp = datetime.now()
-        self.sha256_digest = "not implemented"
-        self.name = name if name else type(self).__name__ + " dataset"
-        self.description = description if description else datetime.now().strftime("%d/%m/%Y %H:%M:%S")
-        self.state = state if state else self.DatasetInternalState()
-        self.aux_handlers = aux_handlers
-        self.root_dir = Path(root_dir)
-
-        if not self.aux_handlers:
-            self.aux_handlers[CPEDatasetHandler] = CPEDatasetHandler(self.auxiliary_datasets_dir)
-            self.aux_handlers[CVEDatasetHandler] = CVEDatasetHandler(self.auxiliary_datasets_dir)
-            self.aux_handlers[CPEMatchDictHandler] = CPEMatchDictHandler(self.auxiliary_datasets_dir)
-            self.aux_handlers[CCSchemeDatasetHandler] = CCSchemeDatasetHandler(self.auxiliary_datasets_dir)
-            self.aux_handlers[ProtectionProfileDatasetHandler] = ProtectionProfileDatasetHandler(
-                self.auxiliary_datasets_dir
-            )
-            self.aux_handlers[CCMaintenanceUpdateDatasetHandler] = CCMaintenanceUpdateDatasetHandler(
-                self.auxiliary_datasets_dir
-            )
+        super().__init__(certs, root_dir, name, description, state, aux_handlers)
+        if aux_handlers is None:
+            self.aux_handlers = {
+                CPEDatasetHandler: CPEDatasetHandler(self.auxiliary_datasets_dir),
+                CVEDatasetHandler: CVEDatasetHandler(self.auxiliary_datasets_dir),
+                CPEMatchDictHandler: CPEMatchDictHandler(self.auxiliary_datasets_dir),
+                CCSchemeDatasetHandler: CCSchemeDatasetHandler(self.auxiliary_datasets_dir),
+                ProtectionProfileDatasetHandler: ProtectionProfileDatasetHandler(self.auxiliary_datasets_dir),
+                CCMaintenanceUpdateDatasetHandler: CCMaintenanceUpdateDatasetHandler(self.auxiliary_datasets_dir),
+            }
 
     def to_pandas(self) -> pd.DataFrame:
         """
@@ -820,14 +810,13 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType):
     # Quite difficult to achieve correct behaviour with MyPy here, opting for ignore
     def __init__(
         self,
-        certs: dict[str, CCMaintenanceUpdate] = {},  # type: ignore
+        certs: dict[str, CCMaintenanceUpdate] | None = None,  # type: ignore
         root_dir: Path = constants.DUMMY_NONEXISTING_PATH,
         name: str = "dataset name",
         description: str = "dataset_description",
         state: CCDataset.DatasetInternalState | None = None,
     ):
-        super().__init__(certs, root_dir, name, description, state)  # type: ignore
-        self.aux_handlers = {}
+        super().__init__(certs, root_dir, name, description, state, aux_handlers={})  # type: ignore
         self.state.meta_sources_parsed = True
 
     @property
diff --git a/src/sec_certs/dataset/cpe.py b/src/sec_certs/dataset/cpe.py
index 3257d2a4..43db8a79 100644
--- a/src/sec_certs/dataset/cpe.py
+++ b/src/sec_certs/dataset/cpe.py
@@ -31,11 +31,11 @@ class CPEDataset(JSONPathDataset, ComplexSerializableType):
 
     def __init__(
         self,
-        cpes: dict[str, CPE] = {},
+        cpes: dict[str, CPE] | None = None,
         json_path: str | Path = constants.DUMMY_NONEXISTING_PATH,
         last_update_timestamp: datetime = datetime.fromtimestamp(0),
     ):
-        self.cpes = cpes
+        self.cpes = cpes if cpes is not None else {}
         self.json_path = Path(json_path)
         self.last_update_timestamp = last_update_timestamp
 
diff --git a/src/sec_certs/dataset/cve.py b/src/sec_certs/dataset/cve.py
index f76211ee..1dbf8b37 100644
--- a/src/sec_certs/dataset/cve.py
+++ b/src/sec_certs/dataset/cve.py
@@ -29,11 +29,11 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
 
     def __init__(
         self,
-        cves: dict[str, CVE] = {},
+        cves: dict[str, CVE] | None = None,
         json_path: str | Path = constants.DUMMY_NONEXISTING_PATH,
         last_update_timestamp: datetime = datetime.fromtimestamp(0),
     ):
-        self.cves = cves
+        self.cves = cves if cves is not None else {}
         self.json_path = Path(json_path)
         self._cpe_uri_to_cve_ids_lookup: dict[str, set[str]] = {}
         self._cves_with_vulnerable_configurations: list[CVE] = []
@@ -120,7 +120,7 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
     def build_lookup_dict(
         self,
         cpe_match_feed: dict,
-        limit_to_cpes: set[CPE] = set(),
+        limit_to_cpes: set[CPE] | None = None,
     ):
         self._cpe_uri_to_cve_ids_lookup = {}
         cpe_uris_of_interest = {x.uri for x in limit_to_cpes} if limit_to_cpes else None
diff --git a/src/sec_certs/dataset/dataset.py b/src/sec_certs/dataset/dataset.py
index c17109f3..03535764 100644
--- a/src/sec_certs/dataset/dataset.py
+++ b/src/sec_certs/dataset/dataset.py
@@ -51,22 +51,24 @@ class Dataset(Generic[CertSubType], ComplexSerializableType, ABC):
 
     def __init__(
         self,
-        certs: dict[str, CertSubType] = {},
+        certs: dict[str, CertSubType] | None = None,
         root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH,
         name: str | None = None,
         description: str = "",
         state: DatasetInternalState | None = None,
-        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] = {},
+        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] | None = None,
     ):
-        self.certs = certs
+        super().__init__()
+        self.certs = certs if certs is not None else {}
 
         self.timestamp = datetime.now()
-        self.sha256_digest = "not implemented"
-        self.name = name if name else type(self).__name__.lower() + "_dataset"
-        self.description = description if description else "No description provided"
+        self.name = name if name else type(self).__name__
+        self.description = description if description else datetime.now().strftime("%d/%m/%Y %H:%M:%S")
         self.state = state if state else self.DatasetInternalState()
-        self.aux_handlers = aux_handlers
         self.root_dir = Path(root_dir)
+        self.aux_handlers = aux_handlers if aux_handlers is not None else {}
+        # Make sure that the auxiliary handlers (if supplied by the user) have the correct root_dir
+        self._set_local_paths()
 
     @property
     def root_dir(self) -> Path:
@@ -222,7 +224,6 @@ class Dataset(Generic[CertSubType], ComplexSerializableType, ABC):
         return {
             "state": self.state,
             "timestamp": self.timestamp,
-            "sha256_digest": self.sha256_digest,
             "name": self.name,
             "description": self.description,
             "n_certs": len(self),
@@ -250,8 +251,9 @@ class Dataset(Generic[CertSubType], ComplexSerializableType, ABC):
         return dset
 
     def _set_local_paths(self) -> None:
-        for handler in self.aux_handlers.values():
-            handler.set_local_paths(self.auxiliary_datasets_dir)
+        if hasattr(self, "aux_handlers"):
+            for handler in self.aux_handlers.values():
+                handler.set_local_paths(self.auxiliary_datasets_dir)
 
     def move_dataset(self, new_root_dir: str | Path) -> None:
         """
diff --git a/src/sec_certs/dataset/fips.py b/src/sec_certs/dataset/fips.py
index 34b1147d..cbe4b6b2 100644
--- a/src/sec_certs/dataset/fips.py
+++ b/src/sec_certs/dataset/fips.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import datetime
 import itertools
 import logging
 import shutil
@@ -59,27 +58,21 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType):
 
     def __init__(
         self,
-        certs: dict[str, FIPSCertificate] = {},
+        certs: dict[str, FIPSCertificate] | None = None,
         root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH,
         name: str | None = None,
         description: str = "",
         state: Dataset.DatasetInternalState | None = None,
-        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] = {},
+        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] | None = None,
     ):
-        self.certs = certs
-        self.timestamp = datetime.datetime.now()
-        self.sha256_digest = "not implemented"
-        self.name = name if name else type(self).__name__ + " dataset"
-        self.description = description if description else datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")
-        self.state = state if state else self.DatasetInternalState()
-        self.aux_handlers = aux_handlers
-        self.root_dir = Path(root_dir)
-
-        if not self.aux_handlers:
-            self.aux_handlers[CPEDatasetHandler] = CPEDatasetHandler(self.auxiliary_datasets_dir)
-            self.aux_handlers[CVEDatasetHandler] = CVEDatasetHandler(self.auxiliary_datasets_dir)
-            self.aux_handlers[FIPSAlgorithmDatasetHandler] = FIPSAlgorithmDatasetHandler(self.auxiliary_datasets_dir)
-            self.aux_handlers[CPEMatchDictHandler] = CPEMatchDictHandler(self.auxiliary_datasets_dir)
+        super().__init__(certs, root_dir, name, description, state, aux_handlers)
+        if aux_handlers is None:
+            self.aux_handlers = {
+                CPEDatasetHandler: CPEDatasetHandler(self.auxiliary_datasets_dir),
+                CVEDatasetHandler: CVEDatasetHandler(self.auxiliary_datasets_dir),
+                FIPSAlgorithmDatasetHandler: FIPSAlgorithmDatasetHandler(self.auxiliary_datasets_dir),
+                CPEMatchDictHandler: CPEMatchDictHandler(self.auxiliary_datasets_dir),
+            }
 
     LIST_OF_CERTS_HTML: Final[dict[str, str]] = {
         "fips_modules_active.html": constants.FIPS_ACTIVE_MODULES_URL,
@@ -112,7 +105,6 @@ class FIPSDataset(Dataset[FIPSCertificate], ComplexSerializableType):
     def _extract_data_from_html_modules(self) -> None:
         """
         Extracts data from html module file
-        :param bool fresh: if all certs should be processed, or only the failed ones. Defaults to True
         """
         logger.info("Extracting data from html modules.")
         certs_to_process = [x for x in self if x.state.module_is_ok_to_analyze()]
diff --git a/src/sec_certs/dataset/fips_algorithm.py b/src/sec_certs/dataset/fips_algorithm.py
index ee7d06a1..c9a8b3c5 100644
--- a/src/sec_certs/dataset/fips_algorithm.py
+++ b/src/sec_certs/dataset/fips_algorithm.py
@@ -21,8 +21,10 @@ logger = logging.getLogger(__name__)
 
 
 class FIPSAlgorithmDataset(JSONPathDataset, ComplexSerializableType):
-    def __init__(self, algs: dict[str, FIPSAlgorithm] = {}, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH):
-        self.algs = algs
+    def __init__(
+        self, algs: dict[str, FIPSAlgorithm] | None = None, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH
+    ):
+        self.algs = algs if algs is not None else {}
         self.json_path = Path(json_path)
         self.alg_number_to_algs: dict[str, set[FIPSAlgorithm]] = {}
 
diff --git a/src/sec_certs/dataset/protection_profile.py b/src/sec_certs/dataset/protection_profile.py
index 9b22ca75..b4200795 100644
--- a/src/sec_certs/dataset/protection_profile.py
+++ b/src/sec_certs/dataset/protection_profile.py
@@ -1,5 +1,4 @@
 import shutil
-from datetime import datetime
 from pathlib import Path
 from typing import ClassVar, Literal
 
@@ -39,21 +38,14 @@ class ProtectionProfileDataset(Dataset[ProtectionProfile], ComplexSerializableTy
 
     def __init__(
         self,
-        certs: dict[str, ProtectionProfile] = {},
+        certs: dict[str, ProtectionProfile] | None = None,
         root_dir: str | Path = constants.DUMMY_NONEXISTING_PATH,
         name: str | None = None,
         description: str = "",
         state: Dataset.DatasetInternalState | None = None,
-        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] = {},
+        aux_handlers: dict[type[AuxiliaryDatasetHandler], AuxiliaryDatasetHandler] | None = None,
     ):
-        self.certs = certs
-        self.timestamp = datetime.now()
-        self.sha256_digest = "not implemented"
-        self.name = name if name else type(self).__name__ + " dataset"
-        self.description = description if description else datetime.now().strftime("%d/%m/%Y %H:%M:%S")
-        self.state = state if state else self.DatasetInternalState()
-        self.aux_handlers = aux_handlers
-        self.root_dir = Path(root_dir)
+        super().__init__(certs, root_dir, name, description, state, aux_handlers)
 
     @property
     def json_path(self) -> Path:
diff --git a/tests/cc/test_cc_dataset.py b/tests/cc/test_cc_dataset.py
index 69d73295..cba06a64 100644
--- a/tests/cc/test_cc_dataset.py
+++ b/tests/cc/test_cc_dataset.py
@@ -78,6 +78,7 @@ def test_download_and_convert_pdfs(toy_dataset: CCDataset, data_dir: Path):
 
 
 @pytest.mark.slow
+@pytest.mark.xfail(reason="May fail due to network issues.")
 def test_from_web():
     dset = CCDataset.from_web()
     assert len(dset) > 6000
diff --git a/tests/cc/test_cc_maintenance_updates.py b/tests/cc/test_cc_maintenance_updates.py
index 176a9244..683c8644 100644
--- a/tests/cc/test_cc_maintenance_updates.py
+++ b/tests/cc/test_cc_maintenance_updates.py
@@ -78,6 +78,7 @@ def test_to_pandas(mu_dset: CCDatasetMaintenanceUpdates):
 
 
 @pytest.mark.slow
+@pytest.mark.xfail(reason="May fail due to network issues.")
 def test_from_web():
     dset = CCDatasetMaintenanceUpdates.from_web()
     assert dset is not None
diff --git a/tests/cc/test_cc_protection_profiles.py b/tests/cc/test_cc_protection_profiles.py
index e231d87c..4cf61fd0 100644
--- a/tests/cc/test_cc_protection_profiles.py
+++ b/tests/cc/test_cc_protection_profiles.py
@@ -72,6 +72,7 @@ def test_get_certs_from_web(pp_data_dir: Path, toy_pp_dataset: ProtectionProfile
 
 
 @pytest.mark.slow
+@pytest.mark.xfail(reason="May fail due to network issues.")
 def test_from_web():
     dset = ProtectionProfileDataset.from_web()
     assert len(dset) > 400
diff --git a/tests/data/cc/analysis/cc_full_dataset.json b/tests/data/cc/analysis/cc_full_dataset.json
index a2149b67..e39620e1 100644
--- a/tests/data/cc/analysis/cc_full_dataset.json
+++ b/tests/data/cc/analysis/cc_full_dataset.json
@@ -9,7 +9,6 @@
         "certs_analyzed": false
     },
     "timestamp": "2022-11-04 16:28:04.282938",
-    "sha256_digest": "not implemented",
     "name": "cc_full_dataset",
     "description": "sample dataset description",
     "n_certs": 1,
diff --git a/tests/data/cc/analysis/reference_dataset.json b/tests/data/cc/analysis/reference_dataset.json
index 4ea176de..1f9f7cbf 100644
--- a/tests/data/cc/analysis/reference_dataset.json
+++ b/tests/data/cc/analysis/reference_dataset.json
@@ -8,7 +8,6 @@
     "certs_analyzed": false
   },
   "timestamp": "2021-11-14 13:18:53.333058",
-  "sha256_digest": "not implemented",
   "name": "test dataset",
   "description": "test dataset for testing references",
   "n_certs": 3,
diff --git a/tests/data/cc/analysis/transitive_vulnerability_dataset.json b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
index 7ed77910..926e0a58 100644
--- a/tests/data/cc/analysis/transitive_vulnerability_dataset.json
+++ b/tests/data/cc/analysis/transitive_vulnerability_dataset.json
@@ -8,7 +8,6 @@
     "certs_analyzed": false
   },
   "timestamp": "2022-04-09 14:40:30",
-  "sha256_digest": "not implemented",
   "name": "test dataset",
   "description": "test dataset for testing transitive vulnerabilities",
   "n_certs": 3,
diff --git a/tests/data/cc/analysis/vulnerable_dataset.json b/tests/data/cc/analysis/vulnerable_dataset.json
index 776db230..d3ecb3ac 100644
--- a/tests/data/cc/analysis/vulnerable_dataset.json
+++ b/tests/data/cc/analysis/vulnerable_dataset.json
@@ -9,7 +9,6 @@
         "certs_analyzed": false
     },
     "timestamp": "2021-04-16 15:05:18.386794",
-    "sha256_digest": "not implemented",
     "name": "cc_full_dataset",
     "description": "sample dataset description",
     "n_certs": 2,
diff --git a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
index d8de0f3a..d9430480 100644
--- a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
+++ b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
@@ -9,7 +9,6 @@
         "certs_analyzed": false
     },
     "timestamp": "2022-11-10 13:44:35.171285",
-    "sha256_digest": "not implemented",
     "name": "maintenance_updates",
     "description": "dataset_description",
     "n_certs": 1,
diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json
index d593f382..b6e071a4 100644
--- a/tests/data/cc/dataset/toy_dataset.json
+++ b/tests/data/cc/dataset/toy_dataset.json
@@ -9,7 +9,6 @@
     "certs_analyzed": false
   },
   "timestamp": "2020-11-16 17:04:14.770153",
-  "sha256_digest": "not implemented",
   "name": "toy dataset",
   "description": "toy dataset description",
   "n_certs": 3,
diff --git a/tests/data/fips/dataset/toy_dataset.json b/tests/data/fips/dataset/toy_dataset.json
index 97edf45e..fb98ff34 100644
--- a/tests/data/fips/dataset/toy_dataset.json
+++ b/tests/data/fips/dataset/toy_dataset.json
@@ -9,7 +9,6 @@
         "certs_analyzed": false
     },
     "timestamp": "2022-11-30 16:38:51.953055",
-    "sha256_digest": "not implemented",
     "name": "FIPSDataset dataset",
     "description": "30/11/2022 16:38:51",
     "n_certs": 22,
diff --git a/tests/data/protection_profiles/dataset.json b/tests/data/protection_profiles/dataset.json
index 7b96cf8c..6fb0b070 100644
--- a/tests/data/protection_profiles/dataset.json
+++ b/tests/data/protection_profiles/dataset.json
@@ -9,7 +9,6 @@
         "certs_analyzed": false
     },
     "timestamp": "2025-01-25 17:39:26.873380",
-    "sha256_digest": "not implemented",
     "name": "ProtectionProfileDataset dataset",
     "description": "25/01/2025 17:39:26",
     "n_certs": 3,
diff --git a/tests/fips/test_fips_iut.py b/tests/fips/test_fips_iut.py
index cdd8c10d..17d15dc8 100644
--- a/tests/fips/test_fips_iut.py
+++ b/tests/fips/test_fips_iut.py
@@ -33,6 +33,7 @@ def test_iut_dataset_from_dumps(data_dir: Path):
     assert len(dset) == 2
 
 
+@pytest.mark.xfail(reason="May fail due to network issues.")
 def test_iut_dataset_from_web():
     assert IUTDataset.from_web()