implement PP tests

2 files changed, 4 insertions, 58 deletions

diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
index afa1969d..125c2265 100644
--- a/src/sec_certs/constants.py
+++ b/src/sec_certs/constants.py
@@ -22,6 +22,7 @@ MIN_CORRECT_CERT_SIZE = 5000
 MIN_FIPS_HTML_SIZE = 64000
 
 MIN_CC_HTML_SIZE = 5000000
+MIN_PP_HTML_SIZE = 200000
 MIN_CC_CSV_SIZE = 700000
 MIN_CC_PP_DATASET_SIZE = 2500000
 
diff --git a/src/sec_certs/dataset/protection_profile.py b/src/sec_certs/dataset/protection_profile.py
index 4a938d1c..0d409611 100644
--- a/src/sec_certs/dataset/protection_profile.py
+++ b/src/sec_certs/dataset/protection_profile.py
@@ -81,10 +81,6 @@ class ProtectionProfileDataset(Dataset[ProtectionProfile], ComplexSerializableTy
         "pp_archived.html": constants.CC_PORTAL_BASE_URL + "/pps/index.cfm?archived=1",
         "pp_collaborative.html": constants.CC_PORTAL_BASE_URL + "/pps/collaborativePP.cfm?cpp=1",
     }
-    CSV_URL = {
-        "pp_active.csv": constants.CC_PORTAL_BASE_URL + "/pps/pps.csv",
-        "pp_archived.csv": constants.CC_PORTAL_BASE_URL + "/pps/pps-archived.csv",
-    }
 
     @property
     def active_html_tuples(self) -> list[tuple[str, Path]]:
@@ -98,14 +94,6 @@ class ProtectionProfileDataset(Dataset[ProtectionProfile], ComplexSerializableTy
     def collaborative_html_tuples(self) -> list[tuple[str, Path]]:
         return [(x, self.web_dir / y) for y, x in self.HTML_URL.items() if "collaborative" in y]
 
-    @property
-    def active_csv_tuples(self) -> list[tuple[str, Path]]:
-        return [(x, self.web_dir / y) for y, x in self.CSV_URL.items() if "active" in y]
-
-    @property
-    def archived_csv_tuples(self) -> list[tuple[str, Path]]:
-        return [(x, self.web_dir / y) for y, x in self.CSV_URL.items() if "archived" in y]
-
     @serialize
     @staged(logger, "Downloading and processing CSV and HTML files of certificates.")
     def get_certs_from_web(
@@ -117,17 +105,10 @@ class ProtectionProfileDataset(Dataset[ProtectionProfile], ComplexSerializableTy
         get_collaborative: bool = True,
     ) -> None:
         if to_download:
-            self._download_csv_html_resources(get_active, get_archived, get_collaborative)
-
-        # TODO: Implement CSV processing if needed. If not, delete this and the corresponding methods. Also get rid of URLs.
-        # logger.info("Adding PPs from CSV to ProtectionProfileDataset.")
-        # csv_certs = self._get_all_certs_from_csv(get_active, get_archived)
-        # self._merge_certs(csv_certs, cert_source="csv")
+            self._download_html_resources(get_active, get_archived, get_collaborative)
 
         logger.info("Adding HTML certificates to ProtectionProfile dataset.")
-        html_certs = self._get_all_certs_from_html(get_active, get_archived, get_collaborative)
-        self._merge_certs(html_certs, cert_source="html")
-
+        self.certs = self._get_all_certs_from_html(get_active, get_archived, get_collaborative)
         logger.info(f"The resulting dataset has {len(self)} certificates.")
 
         if not keep_metadata:
@@ -136,19 +117,6 @@ class ProtectionProfileDataset(Dataset[ProtectionProfile], ComplexSerializableTy
         self._set_local_paths()
         self.state.meta_sources_parsed = True
 
-    def _merge_certs(self, certs: dict[str, ProtectionProfile], cert_source: str | None = None) -> None:
-        """
-        Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates
-        """
-        new_certs = {x.dgst: x for x in certs.values() if x not in self}
-        certs_to_merge = [x for x in certs.values() if x in self]
-        self.certs.update(new_certs)
-
-        for crt in certs_to_merge:
-            self[crt.dgst].merge(crt, cert_source)
-
-        logger.info(f"Added {len(new_certs)} new and merged further {len(certs_to_merge)} certificates to the dataset.")
-
     def _get_all_certs_from_html(
         self, get_active: bool = True, get_archived: bool = True, get_collaborative: bool = True
     ) -> dict[str, ProtectionProfile]:
@@ -167,45 +135,22 @@ class ProtectionProfileDataset(Dataset[ProtectionProfile], ComplexSerializableTy
             new_certs.update(partial_certs)
         return new_certs
 
-    def _get_all_certs_from_csv(
-        self, get_active: bool = True, get_archived: bool = True
-    ) -> dict[str, ProtectionProfile]:
-        csv_sources = list(self.CSV_URL.keys())
-        csv_sources = [x for x in csv_sources if "active" not in x or get_active]
-        csv_sources = [x for x in csv_sources if "archived" not in x or get_archived]
-
-        new_certs = {}
-        for file in csv_sources:
-            partial_certs = self._parse_single_csv(self.web_dir / file)
-            logger.info(f"Parsed {len(partial_certs)} certificates from: {file}")
-            new_certs.update(partial_certs)
-        return new_certs
-
-    def _download_csv_html_resources(
+    def _download_html_resources(
         self, get_active: bool = True, get_archived: bool = True, get_collaborative: bool = True
     ) -> None:
         self.web_dir.mkdir(parents=True, exist_ok=True)
         html_items = []
-        csv_items = []
         if get_active:
             html_items.extend(self.active_html_tuples)
-            csv_items.extend(self.active_csv_tuples)
         if get_archived:
             html_items.extend(self.archived_html_tuples)
-            html_items.extend(self.archived_csv_tuples)
         if get_collaborative:
             html_items.extend(self.collaborative_html_tuples)
 
         html_urls, html_paths = [x[0] for x in html_items], [x[1] for x in html_items]
-        csv_urls, csv_paths = [x[0] for x in csv_items], [x[1] for x in csv_items]
 
         logger.info("Downloading required csv and html files.")
         helpers.download_parallel(html_urls, html_paths)
-        helpers.download_parallel(csv_urls, csv_paths)
-
-    @staticmethod
-    def _parse_single_csv(file: Path) -> dict[str, ProtectionProfile]:
-        return {}
 
     @staticmethod
     def _parse_single_html(file: Path) -> dict[str, ProtectionProfile]: