Add explicit certificate status active/archived

- Status active or archived held as a string variable
- Instead of implicit computation from not-valid-before/after
- The status is computed from the filename of the csv/html source
- The status is serialized into json

2 files changed, 18 insertions, 7 deletions

diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index e42384fb..6f451f01 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -856,7 +856,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         def from_dict(cls, dct: Dict[str, bool]):
             return cls(*tuple(dct.values()))
 
-    def __init__(self, category: str, name: str, manufacturer: str, scheme: str,
+    def __init__(self, status:str, category: str, name: str, manufacturer: str, scheme: str,
                  security_level: Union[str, set], not_valid_before: date,
                  not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str],
                  manufacturer_web: Optional[str],
@@ -866,6 +866,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
                  pdf_data: Optional[PdfData]):
         super().__init__()
 
+        self.status = status
         self.category = category
         self.name = helpers.sanitize_string(name)
         self.manufacturer = helpers.sanitize_string(manufacturer)
@@ -933,7 +934,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         return super(cls, CommonCriteriaCert).from_dict(new_dct)
 
     @classmethod
-    def from_html_row(cls, row: Tag, category: str) -> 'CommonCriteriaCert':
+    def from_html_row(cls, row: Tag, status: str, category: str) -> 'CommonCriteriaCert':
         """
         Creates a CC certificate from html row
         """
@@ -1041,7 +1042,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         maintainances = _get_maintainance_updates(
             maintainance_div) if maintainance_div else set()
 
-        return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link,
+        return cls(status, category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link,
                    st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None, None)
 
     def set_local_paths(self,
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 80eb49bd..351a0f56 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -279,6 +279,11 @@ class CCDataset(Dataset, ComplexSerializableType):
             prim_key = row['category'] + row['cert_name'] + row['report_link']
             return prim_key
 
+        if 'active' in str(file):
+            cert_status = 'active'
+        else:
+            cert_status = 'archived'
+
         csv_header = ['category', 'cert_name', 'manufacturer', 'scheme', 'security_level', 'protection_profiles',
                       'not_valid_before', 'not_valid_after', 'report_link', 'st_link', 'maintainance_date',
                       'maintainance_title', 'maintainance_report_link', 'maintainance_st_link']
@@ -316,7 +321,7 @@ class CCDataset(Dataset, ComplexSerializableType):
                                                                       x.maintainance_report_link,
                                                                       x.maintainance_st_link))
 
-        certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level,
+        certs = {x.dgst: CommonCriteriaCert(cert_status, x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level,
                                             x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv',
                                             None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None), None, None) for
                  x in
@@ -356,7 +361,7 @@ class CCDataset(Dataset, ComplexSerializableType):
                                  date_string[1] + ' ' + time_string
             return datetime.strptime(formatted_datetime, ' %B %d %Y %I:%M %p')
 
-        def _parse_table(soup: BeautifulSoup, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']:
+        def _parse_table(soup: BeautifulSoup, cert_status: str, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']:
             tables = soup.find_all('table', id=table_id)
             assert len(tables) <= 1
 
@@ -374,10 +379,15 @@ class CCDataset(Dataset, ComplexSerializableType):
             # caption_str = str(table.findAll('caption'))
             # n_expected_certs = int(caption_str.split(category_string + ' – ')[1].split(' Certified Products')[0])
             table_certs = {x.dgst: x for x in [
-                CommonCriteriaCert.from_html_row(row, category_string) for row in body]}
+                CommonCriteriaCert.from_html_row(row, cert_status, category_string) for row in body]}
 
             return table_certs
 
+        if 'active' in str(file):
+            cert_status = 'active'
+        else:
+            cert_status = 'archived'
+
         cc_cat_abbreviations = ['AC', 'BP', 'DP', 'DB', 'DD', 'IC', 'KM',
                                 'MD', 'MF', 'NS', 'OS', 'OD', 'DG', 'TC']
         cc_table_ids = ['tbl' + x for x in cc_cat_abbreviations]
@@ -403,7 +413,7 @@ class CCDataset(Dataset, ComplexSerializableType):
 
         certs = {}
         for key, val in cat_dict.items():
-            certs.update(_parse_table(soup, key, val))
+            certs.update(_parse_table(soup, cert_status, key, val))
 
         return certs