New API: delete src attribute of CC cert

7 files changed, 17 insertions, 34 deletions

diff --git a/sec_certs/certificate/common_criteria.py b/sec_certs/certificate/common_criteria.py
index 135029be..c3df885b 100644
--- a/sec_certs/certificate/common_criteria.py
+++ b/sec_certs/certificate/common_criteria.py
@@ -208,7 +208,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
 
     def __init__(self, status: str, category: str, name: str, manufacturer: str, scheme: str,
                  security_level: Union[str, set], not_valid_before: date,
-                 not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str],
+                 not_valid_after: date, report_link: str, st_link: str, cert_link: Optional[str],
                  manufacturer_web: Optional[str],
                  protection_profiles: set,
                  maintainance_updates: set,
@@ -227,7 +227,6 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         self.not_valid_after = helpers.sanitize_date(not_valid_after)
         self.report_link = helpers.sanitize_link(report_link)
         self.st_link = helpers.sanitize_link(st_link)
-        self.src = src
         self.cert_link = helpers.sanitize_link(cert_link)
         self.manufacturer_web = helpers.sanitize_link(manufacturer_web)
         self.protection_profiles = protection_profiles
@@ -261,7 +260,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
                self.heuristics.extracted_versions, self.heuristics.cpe_matches, self.heuristics.verified_cpe_matches, \
                self.heuristics.related_cves
 
-    def merge(self, other: 'CommonCriteriaCert'):
+    def merge(self, other: 'CommonCriteriaCert', other_source: Optional[str] = None):
         """
         Merges with other CC certificate. Assuming they come from different sources, e.g., csv and html.
         Assuming that html source has better protection profiles, they overwrite CSV info
@@ -274,21 +273,16 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         for att, val in vars(self).items():
             if not val:
                 setattr(self, att, getattr(other, att))
-            elif self.src == 'csv' and other.src == 'html' and att == 'protection_profiles':
+            elif other_source == 'html' and att == 'protection_profiles':
                 setattr(self, att, getattr(other, att))
-            elif self.src == 'csv' and other.src == 'html' and att == 'maintainance_updates':
-                # TODO Fix me: This is a simplification. At the moment html contains more reliable info
+            elif other_source == 'html' and att == 'maintainance_updates':
                 setattr(self, att, getattr(other, att))
-            elif att == 'src':
-                pass  # This is expected
             elif att == 'state':
                 setattr(self, att, getattr(other, att))
             else:
                 if getattr(self, att) != getattr(other, att):
                     logger.warning(
                         f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}')
-        if self.src != other.src:
-            self.src = self.src + ' + ' + other.src
 
     @classmethod
     def from_dict(cls, dct: Dict) -> 'CommonCriteriaCert':
@@ -406,7 +400,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
 
         return cls(status, category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after,
                    report_link,
-                   st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None, None, None)
+                   st_link, cert_link, manufacturer_web, protection_profiles, maintainances, None, None, None)
 
     def set_local_paths(self,
                         report_pdf_dir: Optional[Union[str, Path]],
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index d2e69aad..033129d1 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -172,22 +172,18 @@ class CCDataset(Dataset, ComplexSerializableType):
         for cert in self:
             cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir)
 
-    def _merge_certs(self, certs: Dict[str, 'CommonCriteriaCert']):
+    def _merge_certs(self, certs: Dict[str, 'CommonCriteriaCert'], cert_source: Optional[str] = None):
         """
         Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates
         """
-        will_be_added = {}
-        n_merged = 0
-        for crt in certs.values():
-            if crt not in self:
-                will_be_added[crt.dgst] = crt
-            else:
-                self[crt.dgst].merge(crt)
-                n_merged += 1
+        new_certs = {x.dgst: x for x in certs.values() if x not in self}
+        certs_to_merge = [x for x in certs.values() if x in self]
+        self.certs.update(new_certs)
 
-        self.certs.update(will_be_added)
-        logger.info(
-            f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.')
+        for crt in certs_to_merge:
+            self[crt.dgst].merge(crt, cert_source)
+
+        logger.info(f'Added {len(new_certs)} new and merged further {len(certs_to_merge)} certificates to the dataset.')
 
     def download_csv_html_resources(self, get_active: bool = True, get_archived: bool = True):
         self.web_dir.mkdir(parents=True, exist_ok=True)
@@ -231,12 +227,12 @@ class CCDataset(Dataset, ComplexSerializableType):
 
         logger.info('Adding CSV certificates to CommonCriteria dataset.')
         csv_certs = self._get_all_certs_from_csv(get_active, get_archived)
-        self._merge_certs(csv_certs)
+        self._merge_certs(csv_certs, cert_source='csv')
 
         # TODO: Someway along the way, 3 certificates get lost. Investigate and fix.
         logger.info('Adding HTML certificates to CommonCriteria dataset.')
         html_certs = self._get_all_certs_from_html(get_active, get_archived)
-        self._merge_certs(html_certs)
+        self._merge_certs(html_certs, cert_source='html')
 
         logger.info(f'The resulting dataset has {len(self)} certificates.')
 
@@ -319,7 +315,7 @@ class CCDataset(Dataset, ComplexSerializableType):
 
         certs = {
             x.dgst: CommonCriteriaCert(cert_status, x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level,
-                                       x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv',
+                                       x.not_valid_before, x.not_valid_after, x.report_link, x.st_link,
                                        None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None), None, None,
                                        None) for
             x in
diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py
index da77f0ce..43e72605 100644
--- a/sec_certs/dataset/dataset.py
+++ b/sec_certs/dataset/dataset.py
@@ -118,6 +118,6 @@ class Dataset(ABC):
             for p in paths:
                 if p.exists() and p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE:
                     logger.error(f'Corrupted file at: {p}')
-                    # TODO: Delete
+                    p.unlink()
 
 
diff --git a/tests/data/test_cc_oop/fictional_cert.json b/tests/data/test_cc_oop/fictional_cert.json
index 0c4bdc08..0cf17df8 100644
--- a/tests/data/test_cc_oop/fictional_cert.json
+++ b/tests/data/test_cc_oop/fictional_cert.json
@@ -13,7 +13,6 @@
     "not_valid_after": "1900-01-03",
     "report_link": "https://path.to/report/link",
     "st_link": "https://path.to/st/link",
-    "src": "custom",
     "cert_link": "https://path.to/cert/link",
     "manufacturer_web": "https://path.to/manufacturer/web",
     "protection_profiles": [
diff --git a/tests/data/test_cc_oop/toy_dataset.json b/tests/data/test_cc_oop/toy_dataset.json
index fd3016cb..71d715ee 100644
--- a/tests/data/test_cc_oop/toy_dataset.json
+++ b/tests/data/test_cc_oop/toy_dataset.json
@@ -30,7 +30,6 @@
             "not_valid_after": "2025-06-15",
             "report_link": "https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf",
             "st_link": "https://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf",
-            "src": "csv + html",
             "cert_link": "https://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
             "manufacturer_web": "https://www.netiq.com/",
             "protection_profiles": [],
@@ -78,7 +77,6 @@
             "not_valid_after": "2024-11-15",
             "report_link": "https://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf",
             "st_link": "https://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf",
-            "src": "csv + html",
             "cert_link": null,
             "manufacturer_web": "https://www.dreamsecurity.com/",
             "protection_profiles": [
diff --git a/tests/data/test_cpe_cve/vulnerable_dataset.json b/tests/data/test_cpe_cve/vulnerable_dataset.json
index 30a62017..c304fac3 100644
--- a/tests/data/test_cpe_cve/vulnerable_dataset.json
+++ b/tests/data/test_cpe_cve/vulnerable_dataset.json
@@ -30,7 +30,6 @@
             "not_valid_after": null,
             "report_link": "http://www.commoncriteriaportal.org/files/epfiles/0683a_pdf.pdf",
             "st_link": "http://www.commoncriteriaportal.org/files/epfiles/0683b_pdf.pdf",
-            "src": "csv + html",
             "cert_link": null,
             "manufacturer_web": "http://www.ibm.com",
             "protection_profiles": [],
diff --git a/tests/test_cc_oop.py b/tests/test_cc_oop.py
index 255184a2..54fcbd4b 100644
--- a/tests/test_cc_oop.py
+++ b/tests/test_cc_oop.py
@@ -29,7 +29,6 @@ class TestCommonCriteriaOOP(TestCase):
                                           date(2025, 6, 15),
                                           'https://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf',
                                           'https://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf',
-                                          'csv + html',
                                           'https://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf',
                                           'https://www.netiq.com/',
                                           set(),
@@ -48,7 +47,6 @@ class TestCommonCriteriaOOP(TestCase):
                                           date(2024, 11, 15),
                                           'https://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf',
                                           'https://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf',
-                                          'csv + html',
                                           None,
                                           'https://www.dreamsecurity.com/',
                                           {ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0',
@@ -70,7 +68,6 @@ class TestCommonCriteriaOOP(TestCase):
                                                  date(1900, 1, 3),
                                                  'https://path.to/report/link',
                                                  'https://path.to/st/link',
-                                                 'custom',
                                                  'https://path.to/cert/link',
                                                  'https://path.to/manufacturer/web',
                                                  {pp},