From 5cb2c97f66f70bbb8a93f41a1238bdee2ffcae0f Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 13 Nov 2020 09:12:02 +0100 Subject: . --- sec_certs/certificate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index 0058e22e..91b9fb97 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -252,6 +252,4 @@ class CommonCriteriaCert(Certificate): maintainance_div = get_maintainance_div(cells[0]) maintainances = get_maintainance_updates(maintainance_div) if maintainance_div else set() - crt = CommonCriteriaCert(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances) - - return crt + return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances) -- cgit v1.3.1 From 069b98ccaf01432c6163fad26c23371532b1112c Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 16 Nov 2020 09:21:13 +0100 Subject: Possible to get only archived/active/none crts --- sec_certs/dataset.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 1f19ba23..a0b8f030 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -124,25 +124,35 @@ class CCDataset(Dataset): 'cc_pp_archived.csv': 'https://www.commoncriteriaportal.org/pps/pps-archived.csv' } - def get_certs_from_web(self, keep_metadata: bool = True): + def get_certs_from_web(self, to_download=True, keep_metadata: bool = True, get_active=True, get_archived=True): """ Downloads all metadata about certificates from CSV and HTML sources """ self.web_dir.mkdir(parents=True, exist_ok=True) - logging.info('Downloading required csv and html files.') html_items = [(x, self.web_dir / y) for y, x in self.html_products.items()] csv_items = [(x, self.web_dir / y) for y, x in self.csv_products.items()] - helpers.download_parallel(html_items, num_threads=8) - helpers.download_parallel(csv_items, num_threads=8) + + if not get_active: + html_items = [x for x in html_items if 'active' not in str(x[1])] + csv_items = [x for x in csv_items if 'active' not in str(x[1])] + + if not get_archived: + html_items = [x for x in html_items if 'archived' not in str(x[1])] + csv_items = [x for x in csv_items if 'archived' not in str(x[1])] + + if to_download is True: + logging.info('Downloading required csv and html files.') + helpers.download_parallel(html_items, num_threads=8) + helpers.download_parallel(csv_items, num_threads=8) logging.info('Adding CSV certificates to CommonCriteria dataset.') - csv_certs = self.get_all_certs_from_csv() + csv_certs = self.get_all_certs_from_csv(get_active, get_archived) self.merge_certs(csv_certs) # TODO: Someway along the way, 3 certificates get lost. Investigate and fix. logging.info('Adding HTML certificates to CommonCriteria dataset.') - html_certs = self.get_all_certs_from_html() + html_certs = self.get_all_certs_from_html(get_active, get_archived) self.merge_certs(html_certs) logging.info(f'The resulting dataset has {len(self)} certificates.') @@ -150,12 +160,16 @@ class CCDataset(Dataset): if not keep_metadata: shutil.rmtree(self.web_dir) - def get_all_certs_from_csv(self) -> Dict[str, 'CommonCriteriaCert']: + def get_all_certs_from_csv(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: """ Creates dictionary of new certificates from csv sources. """ + csv_sources = self.csv_products.keys() + csv_sources = [x for x in csv_sources if 'active' not in x or get_active] + csv_sources = [x for x in csv_sources if 'archived' not in x or get_archived] + new_certs = {} - for file in self.csv_products: + for file in csv_sources: partial_certs = self.parse_single_csv(self.web_dir / file) logging.info(f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) @@ -201,12 +215,17 @@ class CCDataset(Dataset): certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level, x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv', None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None)) for x in df_base.itertuples()} return certs - def get_all_certs_from_html(self) -> Dict[str, 'CommonCriteriaCert']: + def get_all_certs_from_html(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: """ Prepares dictionary of certificates from all html files. """ + html_sources = self.html_products.keys() + html_sources = [x for x in html_sources if 'active' not in x or get_active] + html_sources = [x for x in html_sources if 'archived' not in x or get_archived] + + new_certs = {} - for file in self.html_products: + for file in html_sources: partial_certs = self.parse_single_html(self.web_dir / file) logging.info(f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) -- cgit v1.3.1 From 629a99b49c4a0f5cf02a44dcc131e9c159c2f5cf Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 16 Nov 2020 11:20:22 +0100 Subject: added deserialization from json --- sec_certs/certificate.py | 33 ++++++++++++++++++++++----------- sec_certs/dataset.py | 34 ++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index 91b9fb97..9ec073c2 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -74,21 +74,29 @@ class CommonCriteriaCert(Certificate): def to_dict(self): return self.__dict__ + @classmethod + def from_dict(cls, dct): + return cls(*tuple(dct.values())) + @dataclass(eq=True, frozen=True) class ProtectionProfile: """ Object for holding protection profiles. """ - name: str - link: Optional[str] + pp_name: str + pp_link: Optional[str] def __post_init__(self): - super().__setattr__('name', helpers.sanitize_string(self.name)) - super().__setattr__('link', helpers.sanitize_link(self.link)) + super().__setattr__('pp_name', helpers.sanitize_string(self.pp_name)) + super().__setattr__('pp_link', helpers.sanitize_link(self.pp_link)) def to_dict(self): return self.__dict__ + @classmethod + def from_dict(cls, dct): + return cls(*tuple(dct.values())) + def __init__(self, category: str, name: str, manufacturer: str, scheme: str, security_level: Union[str, set], not_valid_before: date, not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str], @@ -110,7 +118,7 @@ class CommonCriteriaCert(Certificate): self.cert_link = helpers.sanitize_link(cert_link) self.manufacturer_web = helpers.sanitize_link(manufacturer_web) self.protection_profiles = protection_profiles - self.maintainances = maintainance_updates + self.maintainance_updates = maintainance_updates @property def dgst(self) -> str: @@ -133,10 +141,9 @@ class CommonCriteriaCert(Certificate): setattr(self, att, getattr(other, att)) elif self.src == 'csv' and other.src == 'html' and att == 'protection_profiles': setattr(self, att, getattr(other, att)) - elif att == 'maintainances': - # TODO Fix me: This is a simplification. Basically take the longer list of maintainances as a ground truth. - if len(getattr(self, att)) < len(getattr(other, att)): - setattr(self, att, getattr(other, att)) + elif self.src == 'csv' and other.src == 'html' and att == 'maintainance_updates': + # TODO Fix me: This is a simplification. At the moment html contains more reliable info + setattr(self, att, getattr(other, att)) elif att == 'src': pass # This is expected else: @@ -150,8 +157,12 @@ class CommonCriteriaCert(Certificate): @classmethod def from_dict(cls, dct: dict) -> 'CommonCriteriaCert': - # TODO: Implement me - pass + dct['maintainance_updates'] = set(dct['maintainance_updates']) + dct['protection_profiles'] = set(dct['protection_profiles']) + + args = tuple(dct.values()) + + return cls(*args) @classmethod def from_html_row(cls, row: Tag, category: str) -> 'CommonCriteriaCert': diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index a0b8f030..b50285c5 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -20,25 +20,41 @@ class DatasetJSONEncoder(json.JSONEncoder): return list(obj) if isinstance(obj, date): return str(obj) + if isinstance(obj, Path): + return str(obj) if isinstance(obj, CommonCriteriaCert.ProtectionProfile): return obj.to_dict() if isinstance(obj, CommonCriteriaCert.MaintainanceReport): return obj.to_dict() if isinstance(obj, Dataset): - return list(obj.certs.values()) + return obj.to_dict() return super().default(obj) +class DatasetJSONDecoder(json.JSONDecoder): + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + + def object_hook(self, obj): + if 'root_dir' in obj: # TODO: This is a heavy simplification + return CCDataset.from_dict(obj) + if 'pp_name' in obj and 'pp_link' in obj: + return CommonCriteriaCert.ProtectionProfile.from_dict(obj) + if 'maintainance_date' in obj and 'maintainance_title' in obj and 'maintainance_report_link' in obj and 'maintainance_st_link': + return CommonCriteriaCert.MaintainanceReport.from_dict(obj) + if 'category' in obj: # TODO: This is heavy simplification. + return CommonCriteriaCert.from_dict(obj) + + class Dataset(ABC): def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name', description: str = 'dataset_description'): - self.certs = certs self.root_dir = root_dir - self.timestamp = datetime.now() self.sha256_digest = 'not implemented' self.name = name self.description = description + self.certs = certs def __iter__(self): for cert in self.certs.values(): @@ -59,18 +75,20 @@ class Dataset(ABC): def __str__(self) -> str: return 'Not implemented' - def to_json(self): - pass - def to_csv(self): pass def to_dataframe(self): pass + def to_dict(self): + return {'root_dir': self.root_dir, 'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest, 'name': self.name, + 'description': self.description, 'n_certs': len(self), 'certs': list(self.certs.values())} + @classmethod - def from_json(cls): - pass + def from_dict(cls, dct: Dict): + certs = {x.dgst: x for x in dct['certs']} + return cls(certs, dct['root_dir'], dct['name'], dct['description']) @classmethod def from_csv(cls): -- cgit v1.3.1 From fe9d02cf7191c0ab1fe949cb2fa4770a478c0340 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 16 Nov 2020 11:25:08 +0100 Subject: demo of dataset deserialization --- oop_demo.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/oop_demo.py b/oop_demo.py index 2d76a6c4..455c0ad7 100644 --- a/oop_demo.py +++ b/oop_demo.py @@ -1,4 +1,4 @@ -from sec_certs.dataset import CCDataset, DatasetJSONEncoder +from sec_certs.dataset import CCDataset, DatasetJSONEncoder, DatasetJSONDecoder from pathlib import Path from datetime import datetime import logging @@ -20,6 +20,12 @@ def main(): with open('./debug_dataset/cc_full_dataset.json', 'w') as handle: json.dump(dset, handle, cls=DatasetJSONEncoder, indent=4) + # Load dataset from JSON + with open('./debug_dataset/cc_full_dataset.json', 'r') as handle: + new_dset = json.load(handle, cls=DatasetJSONDecoder) + + assert dset == new_dset + end = datetime.now() logging.info(f'The computation took {(end-start)} seconds.') -- cgit v1.3.1 From 2fe9d4e6ad819d7b7e6c13326890ab2a4c39b14c Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 16 Nov 2020 19:49:08 +0100 Subject: Added basic OOP tests for CC dataset and cert --- sec_certs/dataset.py | 7 +- sec_certs/helpers.py | 4 +- test/data/test_cc_oop/cc_products_active.csv | 3 + test/data/test_cc_oop/cc_products_active.html | 654 ++++++++++++++++++++++++++ test/data/test_cc_oop/fictional_cert.json | 30 ++ test/data/test_cc_oop/toy_dataset.json | 50 ++ test/test_cc_oop.py | 127 +++++ 7 files changed, 872 insertions(+), 3 deletions(-) create mode 100644 test/data/test_cc_oop/cc_products_active.csv create mode 100644 test/data/test_cc_oop/cc_products_active.html create mode 100644 test/data/test_cc_oop/fictional_cert.json create mode 100644 test/data/test_cc_oop/toy_dataset.json create mode 100644 test/test_cc_oop.py diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index b50285c5..7493f615 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -241,7 +241,6 @@ class CCDataset(Dataset): html_sources = [x for x in html_sources if 'active' not in x or get_active] html_sources = [x for x in html_sources if 'archived' not in x or get_archived] - new_certs = {} for file in html_sources: partial_certs = self.parse_single_html(self.web_dir / file) @@ -264,7 +263,11 @@ class CCDataset(Dataset): def parse_table(soup: BeautifulSoup, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']: tables = soup.find_all('table', id=table_id) - assert len(tables) == 1 + assert len(tables) <= 1 + + if not tables: + return {} + table = tables[0] rows = list(table.find_all('tr')) header, footer, body = rows[0], rows[1], rows[2:] diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 35ae843d..da5e73d7 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -58,7 +58,9 @@ def sanitize_string(record: str) -> Union[str, None]: return None else: # TODO: There is a certificate with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I &#38; J' that has to be unescaped twice - return html.unescape(html.unescape(record)).replace('\r\n', ' ').replace('\n', '') + string = html.unescape(html.unescape(record)).replace('\n', '') + return ' '.join(string.split()) + def sanitize_security_levels(record: Union[str, set]) -> set: diff --git a/test/data/test_cc_oop/cc_products_active.csv b/test/data/test_cc_oop/cc_products_active.csv new file mode 100644 index 00000000..2561a372 --- /dev/null +++ b/test/data/test_cc_oop/cc_products_active.csv @@ -0,0 +1,3 @@ +Category,Name,Manufacturer,Scheme,Assurance Level,Protection Profile(s),Certification Date,Archived Date,Certification Report URL,Security Target URL,Maintenance Date,Maintenance Title,Maintenance Report,Maintenance ST +Access Control Devices and Systems,NetIQ Identity Manager 4.7,NetIQ Corporation,SE,"EAL3+,ALC_FLR.2",,06/15/2020,06/15/2025,http://www.commoncriteriaportal.org:443/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf,,,, +Access Control Devices and Systems,Magic SSO V4.0,"Dreamsecurity Co., Ltd.",KR,None,KECS-PP-0822-2017 SSO V1.0,11/15/2019,11/15/2024,http://www.commoncriteriaportal.org:443/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf,,,, diff --git a/test/data/test_cc_oop/cc_products_active.html b/test/data/test_cc_oop/cc_products_active.html new file mode 100644 index 00000000..463fc51a --- /dev/null +++ b/test/data/test_cc_oop/cc_products_active.html @@ -0,0 +1,654 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Certified Products : New CC Portal + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + +
+
+ + + +
+ + +
+ +
+ +
+ + +
+ + +

Certified Products

+ + + + + + + + + + + +
+ + Statistics +     + + Download CSV + +     + Archived + Certified Products + +
+ + + +

+ The Common Criteria Recognition Arrangement covers certificates with claims of compliance against Common + Criteria assurance components of either:  +

+
    +
  1. a collaborative Protection Profile (cPP), developed and maintained in accordance with CCRA Annex K, + with assurance activities selected from Evaluation Assurance Levels up to and including level 4 and ALC_FLR, + developed through an International Technical Community endorsed by the Management Committee; or +
  2. Evaluation Assurance Levels 1 through 2 and ALC_FLR.  +
+

+ Where a CC certificate claims compliance to Evaluation Assurance Level 3 or higher, but does not claim + compliance to a collaborative Protection Profile, then for purposes of mutual recognition under the CCRA, + the CC certificate should be treated as equivalent to Evaluation Assurance Level 2.  +

+

+ The CCDB has approved a resolution to limit the validity of mutually recognized CC certificates over + time.  + Certificates will remain on the CPL for five years.  + Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date + of certificate issuance) will be moved to an Archive list on the CCRA portal, unless the validity period has + been extended using the appropriate procedures.  +

+ + + + + + + +

+ expand/collapse all categories +

+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ProductVendor + Product
Certificate		 + Date Certificate
Issued		 + Certificate
Validity
Expiration
Date		ComplianceScheme
+ This list was generated on Wednesday, November 11, 2020, at 10:47 AM +
+ Access Control Devices and + Systems – 27 Certified Products +
+ + NetIQ Identity Manager 4.7   +
+ +
+ + + + + + +
+ 		+ NetIQ Corporation + + + CCRA Certificate + + 2020-06-152025-06-15 + + + EAL3+ +
ALC_FLR.2 + 		+ + SE – Swedish Certification Body for IT Security FMV/CSEC
SE +
+ + Magic SSO V4.0   +
+ +
+ + + + + + +
+ 		+ Dreamsecurity Co., Ltd. + + +   + + 2019-11-152024-11-15 + + + PP Compliant + + + KR – IT Security Certification Center(ITSCC)
KR +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + \ No newline at end of file diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json new file mode 100644 index 00000000..44a422db --- /dev/null +++ b/test/data/test_cc_oop/fictional_cert.json @@ -0,0 +1,30 @@ +{ + "category": "Sample category", + "name": "Sample certificate name", + "manufacturer": "Sample manufacturer", + "scheme": "Sample scheme", + "security_level": [ + "Sample security level" + ], + "not_valid_before": "1900-01-02", + "not_valid_after": "1900-01-03", + "report_link": "http://path.to/report/link", + "st_link": "http://path.to/st/link", + "src": "custom", + "cert_link": "http://path.to/cert/link", + "manufacturer_web": "http://path.to/manufacturer/web", + "protection_profiles": [ + { + "pp_name": "sample_pp", + "pp_link": "http://sample.pp" + } + ], + "maintainance_updates": [ + { + "maintainance_date": "1900-01-01", + "maintainance_title": "Sample maintainance", + "maintainance_report_link": "https://maintainance.up", + "maintainance_st_link": "https://maintainance.up" + } + ] +} \ No newline at end of file diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json new file mode 100644 index 00000000..49d612b5 --- /dev/null +++ b/test/data/test_cc_oop/toy_dataset.json @@ -0,0 +1,50 @@ +{ + "root_dir": "/fictional/path/to/dataset", + "timestamp": "2020-11-16 17:04:14.770153", + "sha256_digest": "not implemented", + "name": "toy dataset", + "description": "toy dataset description", + "n_certs": 2, + "certs": [ + { + "category": "Access Control Devices and Systems", + "name": "NetIQ Identity Manager 4.7", + "manufacturer": "NetIQ Corporation", + "scheme": "SE", + "security_level": [ + "ALC_FLR.2", + "EAL3+" + ], + "not_valid_before": "2020-06-15", + "not_valid_after": "2025-06-15", + "report_link": "http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf", + "st_link": "http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf", + "src": "csv + html", + "cert_link": "http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf", + "manufacturer_web": "https://www.netiq.com/", + "protection_profiles": [], + "maintainance_updates": [] + }, + { + "category": "Access Control Devices and Systems", + "name": "Magic SSO V4.0", + "manufacturer": "Dreamsecurity Co., Ltd.", + "scheme": "KR", + "security_level": [], + "not_valid_before": "2019-11-15", + "not_valid_after": "2024-11-15", + "report_link": "http://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf", + "st_link": "http://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf", + "src": "csv + html", + "cert_link": null, + "manufacturer_web": "https://www.dreamsecurity.com/", + "protection_profiles": [ + { + "pp_name": "Korean National Protection Profile for Single Sign On V1.0", + "pp_link": "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf" + } + ], + "maintainance_updates": [] + } + ] +} \ No newline at end of file diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py new file mode 100644 index 00000000..c29e90e3 --- /dev/null +++ b/test/test_cc_oop.py @@ -0,0 +1,127 @@ +from unittest import TestCase +from pathlib import Path +from tempfile import TemporaryDirectory, mkstemp +from datetime import date, datetime +import json +import filecmp +import shutil +import os + +from sec_certs.dataset import CCDataset, DatasetJSONDecoder, DatasetJSONEncoder +from sec_certs.certificate import CommonCriteriaCert + + +class TestCommonCriteriaOOP(TestCase): + def setUp(self): + self.test_data_dir = Path(__file__).parent / 'data' / 'test_cc_oop' + self.crt_one = CommonCriteriaCert('Access Control Devices and Systems', + 'NetIQ Identity Manager 4.7', + 'NetIQ Corporation', + 'SE', + {'ALC_FLR.2', + 'EAL3+'}, + date(2020, 6, 15), + date(2025, 6, 15), + 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf', + 'http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf', + 'csv + html', + 'http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf', + 'https://www.netiq.com/', + set(), + set()) + + self.crt_two = CommonCriteriaCert('Access Control Devices and Systems', + 'Magic SSO V4.0', + 'Dreamsecurity Co., Ltd.', + 'KR', + set(), + date(2019, 11, 15), + date(2024, 11, 15), + 'http://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf', + 'http://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf', + 'csv + html', + None, + 'https://www.dreamsecurity.com/', + {CommonCriteriaCert.ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0', + 'http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf')}, + set()) + + pp = CommonCriteriaCert.ProtectionProfile('sample_pp', 'http://sample.pp') + update = CommonCriteriaCert.MaintainanceReport(date(1900, 1, 1), 'Sample maintainance', 'https://maintainance.up', 'https://maintainance.up') + self.fictional_cert = CommonCriteriaCert('Sample category', + 'Sample certificate name', + 'Sample manufacturer', + 'Sample scheme', + {'Sample security level'}, + date(1900, 1, 2), + date(1900, 1, 3), + 'http://path.to/report/link', + 'http://path.to/st/link', + 'custom', + 'http://path.to/cert/link', + 'http://path.to/manufacturer/web', + {pp}, + {update}) + self.template_dataset = CCDataset({self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, Path('/fictional/path/to/dataset'), 'toy dataset', 'toy dataset description') + self.template_dataset.timestamp = datetime(2020, 11, 16, hour=17, minute=4, second=14, microsecond=770153) + + def test_certificate_input_sanity(self): + self.assertEqual(self.crt_one.report_link, + 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf', + 'Report link contains some improperly escaped characters.') + + @staticmethod + def equal_to_json(referential_path, obj): + fd, path = mkstemp() + try: + with os.fdopen(fd, 'w') as handle: + json.dump(obj, handle, cls=DatasetJSONEncoder, indent=4) + + return filecmp.cmp(referential_path, path) + finally: + os.remove(path) + + @staticmethod + def equal_from_json(referential_path, obj): + with open(referential_path, 'r') as handle: + new_obj = json.load(handle, cls=DatasetJSONDecoder) + return obj == new_obj + + def test_cert_to_json(self): + self.assertTrue(self.equal_to_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), + 'The certificate serialized to json differs from a template.') + + def test_dataset_to_json(self): + self.assertTrue(self.equal_to_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset), + 'The dataset serialized to json differs from a template.') + + def test_cert_from_json(self): + self.assertTrue(self.equal_from_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), + 'The certificate serialized from json differs from a template.') + + def test_dataset_from_json(self): + self.assertTrue(self.equal_from_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset), + 'The dataset serialized from json differs from a template.') + + def test_build_empty_dataset(self): + with TemporaryDirectory() as tmp_dir: + dset = CCDataset({}, Path(tmp_dir), 'sample_dataset', 'sample dataset description') + dset.get_certs_from_web(to_download=False, get_archived=False, get_active=False) + self.assertEqual(len(dset), 0, 'The dataset should contain 0 files.') + + def test_build_dataset(self): + with TemporaryDirectory() as tmp_dir: + dataset_path = Path(tmp_dir) + os.mkdir(dataset_path / 'web') + shutil.copyfile(self.test_data_dir / 'cc_products_active.csv', dataset_path / 'web' / 'cc_products_active.csv') + shutil.copyfile(self.test_data_dir / 'cc_products_active.html', dataset_path / 'web' / 'cc_products_active.html') + + dset = CCDataset({}, dataset_path, 'sample_dataset', 'sample dataset description') + dset.get_certs_from_web(keep_metadata=False, to_download=False, get_archived=False, get_active=True) + + self.assertEqual(len(os.listdir(dataset_path)), 0, + 'Meta files (csv, html) were not deleted properly albeit this was explicitly required.') + + self.assertEqual(len(dset), 2, 'The dataset should contain 2 files.') + self.assertTrue(self.crt_one in dset, 'The dataset does not contain the template certificate.') + self.assertEqual(dset, self.template_dataset, 'The loaded dataset does not match the template dataset.') -- cgit v1.3.1 From 88e7b513bb352f42b86aa0c400ed1126e1e9c264 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Mon, 16 Nov 2020 20:03:41 +0100 Subject: sorting keys in list when serializing into json --- sec_certs/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 7493f615..69fcb2c9 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -17,7 +17,7 @@ class DatasetJSONEncoder(json.JSONEncoder): if isinstance(obj, Certificate): return obj.to_dict() if isinstance(obj, set): - return list(obj) + return sorted(list(obj)) if isinstance(obj, date): return str(obj) if isinstance(obj, Path): -- cgit v1.3.1