Add explicit certificate status active/archived

- Status active or archived held as a string variable
- Instead of implicit computation from not-valid-before/after
- The status is computed from the filename of the csv/html source
- The status is serialized into json

5 files changed, 27 insertions, 10 deletions

diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index e42384fb..6f451f01 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -856,7 +856,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         def from_dict(cls, dct: Dict[str, bool]):
             return cls(*tuple(dct.values()))
 
-    def __init__(self, category: str, name: str, manufacturer: str, scheme: str,
+    def __init__(self, status:str, category: str, name: str, manufacturer: str, scheme: str,
                  security_level: Union[str, set], not_valid_before: date,
                  not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str],
                  manufacturer_web: Optional[str],
@@ -866,6 +866,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
                  pdf_data: Optional[PdfData]):
         super().__init__()
 
+        self.status = status
         self.category = category
         self.name = helpers.sanitize_string(name)
         self.manufacturer = helpers.sanitize_string(manufacturer)
@@ -933,7 +934,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         return super(cls, CommonCriteriaCert).from_dict(new_dct)
 
     @classmethod
-    def from_html_row(cls, row: Tag, category: str) -> 'CommonCriteriaCert':
+    def from_html_row(cls, row: Tag, status: str, category: str) -> 'CommonCriteriaCert':
         """
         Creates a CC certificate from html row
         """
@@ -1041,7 +1042,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         maintainances = _get_maintainance_updates(
             maintainance_div) if maintainance_div else set()
 
-        return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link,
+        return cls(status, category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link,
                    st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None, None)
 
     def set_local_paths(self,
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 80eb49bd..351a0f56 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -279,6 +279,11 @@ class CCDataset(Dataset, ComplexSerializableType):
             prim_key = row['category'] + row['cert_name'] + row['report_link']
             return prim_key
 
+        if 'active' in str(file):
+            cert_status = 'active'
+        else:
+            cert_status = 'archived'
+
         csv_header = ['category', 'cert_name', 'manufacturer', 'scheme', 'security_level', 'protection_profiles',
                       'not_valid_before', 'not_valid_after', 'report_link', 'st_link', 'maintainance_date',
                       'maintainance_title', 'maintainance_report_link', 'maintainance_st_link']
@@ -316,7 +321,7 @@ class CCDataset(Dataset, ComplexSerializableType):
                                                                       x.maintainance_report_link,
                                                                       x.maintainance_st_link))
 
-        certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level,
+        certs = {x.dgst: CommonCriteriaCert(cert_status, x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level,
                                             x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv',
                                             None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None), None, None) for
                  x in
@@ -356,7 +361,7 @@ class CCDataset(Dataset, ComplexSerializableType):
                                  date_string[1] + ' ' + time_string
             return datetime.strptime(formatted_datetime, ' %B %d %Y %I:%M %p')
 
-        def _parse_table(soup: BeautifulSoup, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']:
+        def _parse_table(soup: BeautifulSoup, cert_status: str, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']:
             tables = soup.find_all('table', id=table_id)
             assert len(tables) <= 1
 
@@ -374,10 +379,15 @@ class CCDataset(Dataset, ComplexSerializableType):
             # caption_str = str(table.findAll('caption'))
             # n_expected_certs = int(caption_str.split(category_string + ' – ')[1].split(' Certified Products')[0])
             table_certs = {x.dgst: x for x in [
-                CommonCriteriaCert.from_html_row(row, category_string) for row in body]}
+                CommonCriteriaCert.from_html_row(row, cert_status, category_string) for row in body]}
 
             return table_certs
 
+        if 'active' in str(file):
+            cert_status = 'active'
+        else:
+            cert_status = 'archived'
+
         cc_cat_abbreviations = ['AC', 'BP', 'DP', 'DB', 'DD', 'IC', 'KM',
                                 'MD', 'MF', 'NS', 'OS', 'OD', 'DG', 'TC']
         cc_table_ids = ['tbl' + x for x in cc_cat_abbreviations]
@@ -403,7 +413,7 @@ class CCDataset(Dataset, ComplexSerializableType):
 
         certs = {}
         for key, val in cat_dict.items():
-            certs.update(_parse_table(soup, key, val))
+            certs.update(_parse_table(soup, cert_status, key, val))
 
         return certs
 
diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json
index 0bde57bb..80f116e0 100644
--- a/test/data/test_cc_oop/fictional_cert.json
+++ b/test/data/test_cc_oop/fictional_cert.json
@@ -1,5 +1,6 @@
 {
     "_type": "CommonCriteriaCert",
+    "status": "archived",
     "category": "Sample category",
     "name": "Sample certificate name",
     "manufacturer": "Sample manufacturer",
diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json
index bfd08c20..e27076f3 100644
--- a/test/data/test_cc_oop/toy_dataset.json
+++ b/test/data/test_cc_oop/toy_dataset.json
@@ -8,6 +8,7 @@
     "certs": [
         {
             "_type": "CommonCriteriaCert",
+            "status": "active",
             "category": "Access Control Devices and Systems",
             "name": "NetIQ Identity Manager 4.7",
             "manufacturer": "NetIQ Corporation",
@@ -46,6 +47,7 @@
         },
         {
             "_type": "CommonCriteriaCert",
+            "status": "active",
             "category": "Access Control Devices and Systems",
             "name": "Magic SSO V4.0",
             "manufacturer": "Dreamsecurity Co., Ltd.",
diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py
index fbb64a30..7bfac882 100644
--- a/test/test_cc_oop.py
+++ b/test/test_cc_oop.py
@@ -15,7 +15,8 @@ import sec_certs.helpers as helpers
 class TestCommonCriteriaOOP(TestCase):
     def setUp(self):
         self.test_data_dir = Path(__file__).parent / 'data' / 'test_cc_oop'
-        self.crt_one = CommonCriteriaCert('Access Control Devices and Systems',
+        self.crt_one = CommonCriteriaCert('active',
+                                          'Access Control Devices and Systems',
                                           'NetIQ Identity Manager 4.7',
                                           'NetIQ Corporation',
                                           'SE',
@@ -33,7 +34,8 @@ class TestCommonCriteriaOOP(TestCase):
                                           None,
                                           None)
 
-        self.crt_two = CommonCriteriaCert('Access Control Devices and Systems',
+        self.crt_two = CommonCriteriaCert('active',
+                                          'Access Control Devices and Systems',
                                           'Magic SSO V4.0',
                                           'Dreamsecurity Co., Ltd.',
                                           'KR',
@@ -53,7 +55,8 @@ class TestCommonCriteriaOOP(TestCase):
 
         pp = CommonCriteriaCert.ProtectionProfile('sample_pp', 'http://sample.pp')
         update = CommonCriteriaCert.MaintainanceReport(date(1900, 1, 1), 'Sample maintainance', 'https://maintainance.up', 'https://maintainance.up')
-        self.fictional_cert = CommonCriteriaCert('Sample category',
+        self.fictional_cert = CommonCriteriaCert('archived',
+                                                 'Sample category',
                                                  'Sample certificate name',
                                                  'Sample manufacturer',
                                                  'Sample scheme',