diff options
| author | adamjanovsky | 2020-11-16 20:08:25 +0100 |
|---|---|---|
| committer | GitHub | 2020-11-16 20:08:25 +0100 |
| commit | 8fa90fa6312d1cc5e3a270eac14ee74067d8ba2c (patch) | |
| tree | d70e06fde5b473ece1e298b5f0a9e294f521562d | |
| parent | bd772525d02cc17f9c0d2e863698b3ad22a97f79 (diff) | |
| parent | 88e7b513bb352f42b86aa0c400ed1126e1e9c264 (diff) | |
| download | sec-certs-8fa90fa6312d1cc5e3a270eac14ee74067d8ba2c.tar.gz sec-certs-8fa90fa6312d1cc5e3a270eac14ee74067d8ba2c.tar.zst sec-certs-8fa90fa6312d1cc5e3a270eac14ee74067d8ba2c.zip | |
Merge pull request #12 from petrs/cc-oop-tests
oop tests and deserialization from json
| -rw-r--r-- | oop_demo.py | 8 | ||||
| -rw-r--r-- | sec_certs/certificate.py | 37 | ||||
| -rw-r--r-- | sec_certs/dataset.py | 80 | ||||
| -rw-r--r-- | sec_certs/helpers.py | 4 | ||||
| -rw-r--r-- | test/data/test_cc_oop/cc_products_active.csv | 3 | ||||
| -rw-r--r-- | test/data/test_cc_oop/cc_products_active.html | 654 | ||||
| -rw-r--r-- | test/data/test_cc_oop/fictional_cert.json | 30 | ||||
| -rw-r--r-- | test/data/test_cc_oop/toy_dataset.json | 50 | ||||
| -rw-r--r-- | test/test_cc_oop.py | 127 |
9 files changed, 957 insertions, 36 deletions
diff --git a/oop_demo.py b/oop_demo.py index 2d76a6c4..455c0ad7 100644 --- a/oop_demo.py +++ b/oop_demo.py @@ -1,4 +1,4 @@ -from sec_certs.dataset import CCDataset, DatasetJSONEncoder +from sec_certs.dataset import CCDataset, DatasetJSONEncoder, DatasetJSONDecoder from pathlib import Path from datetime import datetime import logging @@ -20,6 +20,12 @@ def main(): with open('./debug_dataset/cc_full_dataset.json', 'w') as handle: json.dump(dset, handle, cls=DatasetJSONEncoder, indent=4) + # Load dataset from JSON + with open('./debug_dataset/cc_full_dataset.json', 'r') as handle: + new_dset = json.load(handle, cls=DatasetJSONDecoder) + + assert dset == new_dset + end = datetime.now() logging.info(f'The computation took {(end-start)} seconds.') diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index 0058e22e..9ec073c2 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -74,21 +74,29 @@ class CommonCriteriaCert(Certificate): def to_dict(self): return self.__dict__ + @classmethod + def from_dict(cls, dct): + return cls(*tuple(dct.values())) + @dataclass(eq=True, frozen=True) class ProtectionProfile: """ Object for holding protection profiles. """ - name: str - link: Optional[str] + pp_name: str + pp_link: Optional[str] def __post_init__(self): - super().__setattr__('name', helpers.sanitize_string(self.name)) - super().__setattr__('link', helpers.sanitize_link(self.link)) + super().__setattr__('pp_name', helpers.sanitize_string(self.pp_name)) + super().__setattr__('pp_link', helpers.sanitize_link(self.pp_link)) def to_dict(self): return self.__dict__ + @classmethod + def from_dict(cls, dct): + return cls(*tuple(dct.values())) + def __init__(self, category: str, name: str, manufacturer: str, scheme: str, security_level: Union[str, set], not_valid_before: date, not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str], @@ -110,7 +118,7 @@ class CommonCriteriaCert(Certificate): self.cert_link = helpers.sanitize_link(cert_link) self.manufacturer_web = helpers.sanitize_link(manufacturer_web) self.protection_profiles = protection_profiles - self.maintainances = maintainance_updates + self.maintainance_updates = maintainance_updates @property def dgst(self) -> str: @@ -133,10 +141,9 @@ class CommonCriteriaCert(Certificate): setattr(self, att, getattr(other, att)) elif self.src == 'csv' and other.src == 'html' and att == 'protection_profiles': setattr(self, att, getattr(other, att)) - elif att == 'maintainances': - # TODO Fix me: This is a simplification. Basically take the longer list of maintainances as a ground truth. - if len(getattr(self, att)) < len(getattr(other, att)): - setattr(self, att, getattr(other, att)) + elif self.src == 'csv' and other.src == 'html' and att == 'maintainance_updates': + # TODO Fix me: This is a simplification. At the moment html contains more reliable info + setattr(self, att, getattr(other, att)) elif att == 'src': pass # This is expected else: @@ -150,8 +157,12 @@ class CommonCriteriaCert(Certificate): @classmethod def from_dict(cls, dct: dict) -> 'CommonCriteriaCert': - # TODO: Implement me - pass + dct['maintainance_updates'] = set(dct['maintainance_updates']) + dct['protection_profiles'] = set(dct['protection_profiles']) + + args = tuple(dct.values()) + + return cls(*args) @classmethod def from_html_row(cls, row: Tag, category: str) -> 'CommonCriteriaCert': @@ -252,6 +263,4 @@ class CommonCriteriaCert(Certificate): maintainance_div = get_maintainance_div(cells[0]) maintainances = get_maintainance_updates(maintainance_div) if maintainance_div else set() - crt = CommonCriteriaCert(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances) - - return crt + return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances) diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 1f19ba23..69fcb2c9 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -17,28 +17,44 @@ class DatasetJSONEncoder(json.JSONEncoder): if isinstance(obj, Certificate): return obj.to_dict() if isinstance(obj, set): - return list(obj) + return sorted(list(obj)) if isinstance(obj, date): return str(obj) + if isinstance(obj, Path): + return str(obj) if isinstance(obj, CommonCriteriaCert.ProtectionProfile): return obj.to_dict() if isinstance(obj, CommonCriteriaCert.MaintainanceReport): return obj.to_dict() if isinstance(obj, Dataset): - return list(obj.certs.values()) + return obj.to_dict() return super().default(obj) +class DatasetJSONDecoder(json.JSONDecoder): + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + + def object_hook(self, obj): + if 'root_dir' in obj: # TODO: This is a heavy simplification + return CCDataset.from_dict(obj) + if 'pp_name' in obj and 'pp_link' in obj: + return CommonCriteriaCert.ProtectionProfile.from_dict(obj) + if 'maintainance_date' in obj and 'maintainance_title' in obj and 'maintainance_report_link' in obj and 'maintainance_st_link': + return CommonCriteriaCert.MaintainanceReport.from_dict(obj) + if 'category' in obj: # TODO: This is heavy simplification. + return CommonCriteriaCert.from_dict(obj) + + class Dataset(ABC): def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name', description: str = 'dataset_description'): - self.certs = certs self.root_dir = root_dir - self.timestamp = datetime.now() self.sha256_digest = 'not implemented' self.name = name self.description = description + self.certs = certs def __iter__(self): for cert in self.certs.values(): @@ -59,18 +75,20 @@ class Dataset(ABC): def __str__(self) -> str: return 'Not implemented' - def to_json(self): - pass - def to_csv(self): pass def to_dataframe(self): pass + def to_dict(self): + return {'root_dir': self.root_dir, 'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest, 'name': self.name, + 'description': self.description, 'n_certs': len(self), 'certs': list(self.certs.values())} + @classmethod - def from_json(cls): - pass + def from_dict(cls, dct: Dict): + certs = {x.dgst: x for x in dct['certs']} + return cls(certs, dct['root_dir'], dct['name'], dct['description']) @classmethod def from_csv(cls): @@ -124,25 +142,35 @@ class CCDataset(Dataset): 'cc_pp_archived.csv': 'https://www.commoncriteriaportal.org/pps/pps-archived.csv' } - def get_certs_from_web(self, keep_metadata: bool = True): + def get_certs_from_web(self, to_download=True, keep_metadata: bool = True, get_active=True, get_archived=True): """ Downloads all metadata about certificates from CSV and HTML sources """ self.web_dir.mkdir(parents=True, exist_ok=True) - logging.info('Downloading required csv and html files.') html_items = [(x, self.web_dir / y) for y, x in self.html_products.items()] csv_items = [(x, self.web_dir / y) for y, x in self.csv_products.items()] - helpers.download_parallel(html_items, num_threads=8) - helpers.download_parallel(csv_items, num_threads=8) + + if not get_active: + html_items = [x for x in html_items if 'active' not in str(x[1])] + csv_items = [x for x in csv_items if 'active' not in str(x[1])] + + if not get_archived: + html_items = [x for x in html_items if 'archived' not in str(x[1])] + csv_items = [x for x in csv_items if 'archived' not in str(x[1])] + + if to_download is True: + logging.info('Downloading required csv and html files.') + helpers.download_parallel(html_items, num_threads=8) + helpers.download_parallel(csv_items, num_threads=8) logging.info('Adding CSV certificates to CommonCriteria dataset.') - csv_certs = self.get_all_certs_from_csv() + csv_certs = self.get_all_certs_from_csv(get_active, get_archived) self.merge_certs(csv_certs) # TODO: Someway along the way, 3 certificates get lost. Investigate and fix. logging.info('Adding HTML certificates to CommonCriteria dataset.') - html_certs = self.get_all_certs_from_html() + html_certs = self.get_all_certs_from_html(get_active, get_archived) self.merge_certs(html_certs) logging.info(f'The resulting dataset has {len(self)} certificates.') @@ -150,12 +178,16 @@ class CCDataset(Dataset): if not keep_metadata: shutil.rmtree(self.web_dir) - def get_all_certs_from_csv(self) -> Dict[str, 'CommonCriteriaCert']: + def get_all_certs_from_csv(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: """ Creates dictionary of new certificates from csv sources. """ + csv_sources = self.csv_products.keys() + csv_sources = [x for x in csv_sources if 'active' not in x or get_active] + csv_sources = [x for x in csv_sources if 'archived' not in x or get_archived] + new_certs = {} - for file in self.csv_products: + for file in csv_sources: partial_certs = self.parse_single_csv(self.web_dir / file) logging.info(f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) @@ -201,12 +233,16 @@ class CCDataset(Dataset): certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level, x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv', None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None)) for x in df_base.itertuples()} return certs - def get_all_certs_from_html(self) -> Dict[str, 'CommonCriteriaCert']: + def get_all_certs_from_html(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: """ Prepares dictionary of certificates from all html files. """ + html_sources = self.html_products.keys() + html_sources = [x for x in html_sources if 'active' not in x or get_active] + html_sources = [x for x in html_sources if 'archived' not in x or get_archived] + new_certs = {} - for file in self.html_products: + for file in html_sources: partial_certs = self.parse_single_html(self.web_dir / file) logging.info(f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) @@ -227,7 +263,11 @@ class CCDataset(Dataset): def parse_table(soup: BeautifulSoup, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']: tables = soup.find_all('table', id=table_id) - assert len(tables) == 1 + assert len(tables) <= 1 + + if not tables: + return {} + table = tables[0] rows = list(table.find_all('tr')) header, footer, body = rows[0], rows[1], rows[2:] diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 35ae843d..da5e73d7 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -58,7 +58,9 @@ def sanitize_string(record: str) -> Union[str, None]: return None else: # TODO: There is a certificate with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I &#38; J' that has to be unescaped twice - return html.unescape(html.unescape(record)).replace('\r\n', ' ').replace('\n', '') + string = html.unescape(html.unescape(record)).replace('\n', '') + return ' '.join(string.split()) + def sanitize_security_levels(record: Union[str, set]) -> set: diff --git a/test/data/test_cc_oop/cc_products_active.csv b/test/data/test_cc_oop/cc_products_active.csv new file mode 100644 index 00000000..2561a372 --- /dev/null +++ b/test/data/test_cc_oop/cc_products_active.csv @@ -0,0 +1,3 @@ +Category,Name,Manufacturer,Scheme,Assurance Level,Protection Profile(s),Certification Date,Archived Date,Certification Report URL,Security Target URL,Maintenance Date,Maintenance Title,Maintenance Report,Maintenance ST +Access Control Devices and Systems,NetIQ Identity Manager 4.7,NetIQ Corporation,SE,"EAL3+,ALC_FLR.2",,06/15/2020,06/15/2025,http://www.commoncriteriaportal.org:443/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf,,,, +Access Control Devices and Systems,Magic SSO V4.0,"Dreamsecurity Co., Ltd.",KR,None,KECS-PP-0822-2017 SSO V1.0,11/15/2019,11/15/2024,http://www.commoncriteriaportal.org:443/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf,,,, diff --git a/test/data/test_cc_oop/cc_products_active.html b/test/data/test_cc_oop/cc_products_active.html new file mode 100644 index 00000000..463fc51a --- /dev/null +++ b/test/data/test_cc_oop/cc_products_active.html @@ -0,0 +1,654 @@ +<!--Manticore2004-->
+<!--Manticore2004-->
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<!-- no action (no SSL on dev box) -->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+
+<head>
+ <meta name="description" content="" />
+ <meta name="keywords" content="" />
+ <meta name="classification" content="Internet" />
+ <meta name="distribution" content="Global" />
+ <meta name="rating" content="Safe For Kids" />
+ <meta name="copyright" content="public domain" />
+ <meta name="language" content="en" />
+
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+ <meta http-equiv="Accept-Encoding" content="compress, gzip" />
+
+
+ <title>Certified Products : New CC Portal</title>
+
+
+ <link href="/assets/css/main.css" rel="stylesheet" type="text/css" />
+ <link href="/assets/css/print.css" rel="stylesheet" type="text/css" media="print" />
+ <link href="/assets/js/jQuery/themes/base/jquery.ui.all.css" rel="stylesheet" type="text/css" />
+ <link href="/assets/js/jQuery/plugins/tabs/tabs.css" rel="stylesheet" type="text/css" />
+ <!--[if lt IE 7]>
+ <link href="/assets/css/ie-lt-7.css" rel="stylesheet" type="text/css" />
+<![endif]-->
+ <!--[if gte IE 7]>
+ <link href="/assets/css/ie7.css" rel="stylesheet" type="text/css" />
+<![endif]-->
+
+
+ <script src="/assets/js/jQuery/js/jquery-1.7.2.min.js" type="text/javascript"></script>
+ <script src="/assets/js/jQuery/js/jquery-ui-1.8.4.custom.min.js" type="text/javascript"></script>
+ <script src="/assets/js/jQuery/plugins/metadata/jquery.metadata.js" type="text/javascript"></script>
+
+ <script src="/assets/js/jQuery/plugins/tabs/jquery.tools.min.js" type="text/javascript"></script>
+
+
+ <!-- jstablesorter sppt -->
+ <script src="/assets/js/jQuery/plugins/tablesorter/jquery.tablesorter.min.js" type="text/javascript"></script>
+ <link href="/assets/js/jQuery/plugins/tablesorter/css/style.css" rel="stylesheet" type="text/css" />
+ <!-- END jstablesorter sppt -->
+
+
+
+
+ <link rel="alternate" title="Protection Profiles" href="../rss/pps.xml" type="application/rss+xml" />
+
+ <link rel="Shortcut Icon" type="image/ico" href="/assets/images/cc.ico" />
+
+
+ <!-- assets/includes/SQL_Check.cfm -->
+ <!-- ??? ???????? -->
+ <!-- 2012.03.15 - 2012.XX.XX -->
+
+ <!-- This page creates and manages a list of IP addresses to be blocked due -->
+ <!-- to dangerous requests which may be SQL Injection Attacks -->
+ <!-- Included in the site's std header: assets/includes/header.cfm -->
+ <!--Corregidor-->
+
+
+
+
+
+
+
+
+
+ <!--Corregidor-->
+
+
+
+
+
+
+
+
+ <!-- 2012.03.15 - 2012.XX.XX - Initial dev//??? -->
+ <!-- 2012.07.11 - Last mod before 2016//??? -->
+ <!-- 2016.06.23 - Cleanup, minor edits, and comments added//KC -->
+
+
+</head>
+
+
+<!-- 2017.09.07 - Adjusted jstablesorter sppt//KC -->
+
+
+
+<body>
+
+
+ <div id="wrap">
+
+
+ <div id="banner">
+ <p class="topNav">
+ <a href="/sitemap/" title="View the sitemap"> Sitemap</a> |
+
+ <a href="/contact/message/" title="Contact us">Contact</a>
+
+ </p>
+
+ <!-- Google CSE Search Box Begins -->
+
+ <div id="searchbox">
+ <form id="016233930414485990345:f_zj6spfpx4" action="/search/" style="margin:0;padding:0">
+ <span style="white-space:nowrap">
+ <input type="hidden" name="cx" value="016233930414485990345:f_zj6spfpx4" />
+ <input type="hidden" name="cof" value="FORID:11" />
+ <input type="hidden" name="ie" value="UTF-8" />
+ <input name="q" type="text" size="30" />
+ <input type="submit" name="sa" value="Search" />
+ </span>
+ </form>
+ </div>
+ <script type="text/javascript"
+ src="//www.google.com/coop/cse/brand?form=searchbox_016233930414485990345:f_zj6spfpx4"></script>
+
+ <!-- Google CSE Search Box Ends -->
+
+ </div>
+ <div id="logo">
+
+ <p>
+
+ <a href="https://www.commoncriteriaportal.org/members/account/login/">Login <img
+ src="/assets/images/icon_arrow.png" /> </a>
+
+
+ </p>
+ </div>
+ <!--[if gte IE 9]>
+ <style type="text/css">
+ .gradient {
+ filter: none;
+ }$(function() {
+
+ $( "#menutabs" ).menutabs();
+
+
+ </style>
+<![endif]-->
+ <!-- >
+ <style type="text/css">
+ .trad-blink {
+ animation: blinker 1.5s cubic-bezier(.5, 0, 1, 1) infinite alternate;
+ }
+ @keyframes blinker {
+ from { opacity: 1; }
+ to { opacity: .2;}
+ }
+ </style>
+< -->
+
+ <div id="menutabs">
+
+ <ul class="menutabs menuitem">
+ <li><a href="/news/" title="News">
+ NEWS</a></li>
+ <li class="inline-code trad-blink"> <a href="/iccc/" title="ICCC">
+ ICCC</a></li>
+ <li>
+ <a href="/pps/" title="Protection Profiles">
+ PROTECTION PROFILES </a> </li>
+ <li>
+ <a href="/pps/?cpp=1" title="Collaborative Protection Profiles">
+ COLLABORATIVE PPS </a> </li>
+
+ <li> <a href="/products/" title="Certified Products">
+ CERTIFIED PRODUCTS </a></li>
+ <li>
+ <a href="/communities/index.cfm" title="Technology Communities">
+ TECHNICAL COMMUNITIES</a></li>
+ <li>
+ <a href="/cc/">
+ PUBLICATIONS</a> </li>
+ <li><a href="/ccra/index.cfm">
+ ABOUT THE CC</a></li>
+ <li><a href="/">HOME</a></li>
+
+ </ul>
+
+
+
+
+ </div>
+
+
+ <div id="slides">
+ <div class="slides_container">
+
+
+
+ <div><img src="/assets/images/slides/certified_products_banner.jpg"></div>
+
+
+ <br />
+
+ </div>
+
+ </div>
+
+ <!--[if gte IE 9]>
+ <style type="text/css">
+ .gradient {
+ filter: none;
+ }
+ </style>
+<![endif]-->
+ <div id="content">
+
+
+ <h1 id="page_title">Certified Products</h1>
+
+
+
+
+ <!-- --------------------------------------------------------------------------------------------------------------
+ For those of you poking around in the page source, the following URL parameters are available on this page:
+ expand - auto-expand the tables after the page loads
+ names - display filenames for the certification report and security target files for easier (F)inding
+--------------------------------------------------------------------------------------------------------------- -->
+
+
+ <style type="text/css">
+ .gridTable thead,
+ .gridTable tbody,
+ .gridTable tfoot {
+ display: none
+ }
+
+ .gridTable caption a {
+ color: #333;
+ text-decoration: none
+ }
+
+ .gridTable caption {
+ text-align: left !important;
+ background: #fff
+ }
+ </style>
+
+ <script>
+ $(document).ready(function () {
+
+ $("a#toggle").click().toggle(
+ function () { openAll() },
+ function () { closeAll() }
+ );
+ $("a.toggle").click().toggle(
+ function () { openTbl(this.name); },
+ function () { closeTbl(this.name); }
+ );
+
+ function openAll() {
+ $(".gridTable caption").css({ 'background': '#333' });
+ $(".gridTable caption a").css({ 'color': '#fff' });
+ $(".gridTable thead").show();
+ $(".gridTable tbody").show();
+ $(".gridTable tfoot").show();
+ $("a[id^='toggle'] img").attr("src", "/assets/images/minicon.gif");
+ }
+ function closeAll() {
+ $(".gridTable caption").css({ 'background': '#fff' });
+ $(".gridTable caption a").css({ 'color': '#333' });
+ $(".gridTable thead").hide();
+ $(".gridTable tbody").hide();
+ $(".gridTable tfoot").hide();
+ $("a[id^='toggle'] img").attr("src", "/assets/images/plusicon.gif");
+ }
+ function openTbl(tbl) {
+ $(".gridTable#tbl" + tbl + " caption").css({ 'background': '#333' });
+ $(".gridTable#tbl" + tbl + " caption a").css({ 'color': '#fff' });
+ $(".gridTable#tbl" + tbl + " thead").show();
+ $(".gridTable#tbl" + tbl + " tbody").show();
+ $(".gridTable#tbl" + tbl + " tfoot").show();
+ $("a#toggle" + tbl + " img").attr("src", "/assets/images/minicon.gif");
+ }
+ function closeTbl(tbl) {
+ $(".gridTable#tbl" + tbl + " caption").css({ 'background': '#fff' });
+ $(".gridTable#tbl" + tbl + " caption a").css({ 'color': '#333' });
+ $(".gridTable#tbl" + tbl + " thead").hide();
+ $(".gridTable#tbl" + tbl + " tbody").hide();
+ $(".gridTable#tbl" + tbl + " tfoot").hide();
+ $("a#toggle" + tbl + " img").attr("src", "/assets/images/plusicon.gif");
+ }
+
+ function showProgress() {
+ $('#progress').css({ 'display': 'block' });
+ $('#content').attr('disabled', true);
+ }
+ function hideProgress() {
+ $('#progress').css({ 'display': 'none' });
+ $('#content').removeAttr('disabled');
+ }
+ });
+ </script>
+
+ <div class="noprint" style="float:right;margin-top:-3.5em;margin-right:1em;white-space:nowrap">
+
+ <a href="./stats/" title="View the Certified Products Statistics" class="button2">Statistics</a>
+
+ <a href="certified_products.csv" title="Download the Certified Products List as a CSV file" class="button2">
+ <img src="/assets/images/icon_csv.gif" height="16" width="15" alt="" align="top"> Download CSV
+ </a>
+
+ <a href="index.cfm?archived=1" title="View the Archived Certified Products List" class="button2">Archived
+ Certified Products</a>
+
+ </div>
+
+
+ <!-- Blurb for the active list -->
+ <p>
+ The Common Criteria Recognition Arrangement covers certificates with claims of compliance against Common
+ Criteria assurance components of either:
+ </p>
+ <ol>
+ <li /> a collaborative Protection Profile (cPP), developed and maintained in accordance with CCRA Annex K,
+ with assurance activities selected from Evaluation Assurance Levels up to and including level 4 and ALC_FLR,
+ developed through an International Technical Community endorsed by the Management Committee; or
+ <li /> Evaluation Assurance Levels 1 through 2 and ALC_FLR.
+ </ol>
+ <p class="b">
+ Where a CC certificate claims compliance to Evaluation Assurance Level 3 or higher, but does not claim
+ compliance to a collaborative Protection Profile, then for purposes of mutual recognition under the CCRA,
+ the CC certificate should be treated as equivalent to Evaluation Assurance Level 2.
+ </p>
+ <p>
+ The CCDB has approved a resolution to limit the validity of mutually recognized CC certificates over
+ time.
+ Certificates will remain on the CPL for five years.
+ Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date
+ of certificate issuance) will be moved to an Archive list on the CCRA portal, unless the validity period has
+ been extended using the appropriate procedures.
+ </p>
+ <!--
+ <p>
+ The CCDB has approved a resolution to limit the validity of mutually recognized CC certificates over time.
+ Certificates will remain on the CPL for five years.
+ Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date of certificate issuance) will be moved to an Archive list on the CCRA portal
+ </p>
+ <p>
+ The CCDB has approved a resolution to limit the validity of mutually recognized CC certificates over time.
+ Certificates will remain on the CPL for five years.
+ Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date of certificate issuance) will be moved to an Archive list on the CCRA portal.
+ Certificates fully recognized under the CCRA must claim conformance to either a collaborative Protection Profile (cPP) or EAL1-2.
+ <p>
+ -->
+
+
+
+
+
+
+ <h4 id="operations">
+ <a href="#" id="toggle">expand/collapse all categories</a>
+ </h4>
+
+
+ <div id="white">
+
+
+
+ <table class="tablesorter gridTable" id="tblAC" style="width:97%;margin-left: 1em;margin-top:-.5em">
+
+ <thead>
+ <tr>
+ <!-- zxcv class="title2" -->
+
+ <th id="col_1" class="text " style=";text-align:center;width:45%;max-width:45%">Product</th>
+
+
+ <th id="col_2" class="text " style=";text-align:center;width:10%;max-width:10%">Vendor</th>
+
+
+ <th id="col_3" class="text " style=";text-align:center;width:10%;max-width:10%">
+ Product<br />Certificate</th>
+
+
+ <th id="col_4" class="date-iso sortfirstasc " style=";text-align:center;width:10%;max-width:10%">
+ Date Certificate<br />Issued</th>
+
+
+ <th id="col_5" class="date-iso sortfirstasc " style=";text-align:center;width:10%;max-width:10%">
+ Certificate<br />Validity<br />Expiration<br />Date</th>
+
+
+ <th id="col_6" class="text " style=";text-align:center;width:5%;max-width:5%">Compliance</th>
+
+
+ <th id="col_7" class="text last_th" style=";text-align:center;width:10%;max-width:10%">Scheme</th>
+
+
+ </tr>
+ </thead>
+ <!-- 2017.09.06 - Copied to CC Portal from NIAP site//KC -->
+ <!-- 2017.09.06 - Converted tabs to spaces//KC -->
+
+ <tfoot class="hilite7">
+ <!-- hilite1 -->
+ <tr class="">
+ <td colspan="7" class="txt1 c i">
+ This list was generated on Wednesday, November 11, 2020, at 10:47 AM
+ </td>
+ </tr>
+ </tfoot>
+
+ <!-- 2017.09.06 - Copied to CC Portal from NIAP site + mods//KC -->
+
+
+ <caption style="font-weight:bold;font-size:larger">
+ <a name="AC" href="#AC" id="toggleAC" class="toggle"><img src="/assets/images/plusicon.gif" border="0"
+ height="11" width="11" class="toggle" style="margin-right:.5em" />Access Control Devices and
+ Systems – 27 Certified Products</a>
+ </caption>
+ <thead>
+
+
+ </thead>
+ <tfoot></tfoot>
+ <tbody>
+
+ <tr class="">
+ <td class="b">
+ <!-- ID: 2020.1111 -->
+ NetIQ Identity Manager 4.7<a name="2020.1111" style="text-decoration:none;"> </a><span
+ title="2020.1111"> </span>
+ <div class="none">
+ <!-- <a href="https://www.netiq.com/" title="Vendor's web site" target="_blank">NetIQ Corporation</a> -->
+ <img src="/assets/images/spacer.gif" width="600" height="1" class="" align="top"> <br />
+ <!-- ------ ------ ------ Document Links ------ ------ ------ -->
+ <div style="line-height:2.5em">
+ <a href="/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf"
+ title="Certification Report: Certification Report - NetIQ® Identity Manager 4.7.pdf"
+ target="_blank" class="button2">Certification Report</a>
+ <a href="/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf"
+ title="Security Target: ST - NetIQ Identity Manager 4.7.pdf" target="_blank"
+ class="button2">Security Target</a>
+
+ </div>
+ <!-- ------ ------ ------ END Document Links ------ ------ ------ -->
+ <!-- ------ ------ ------ Product Updates ------ ------ ------ -->
+
+ <!-- ------ ------ ------ END Product Updates ------ ------ ------ -->
+ </div>
+ </td>
+ <!--end-product-cell-->
+ <td>
+ <a href="https://www.netiq.com/" title="Vendor's web site"
+ target="_blank">NetIQ Corporation</a>
+ </td>
+ <td>
+
+ <a href="/files/epfiles/Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf"
+ title="Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf" target="_blank"
+ class="button2">CCRA Certificate</a>
+
+ </td>
+ <td style="text-align:center"><span title="2020.1111">2020-06-15</span></td>
+
+ <td style="text-align:center">2025-06-15</td>
+
+
+ <td>
+
+
+ EAL3+
+ <br />ALC_FLR.2
+ </td>
+
+ <td style="text-align:center">
+ <a href="/ccra/members/#SE">
+ <img src="/assets/images/flags/se.png"
+ alt="SE – Swedish Certification Body for IT Security FMV/CSEC"
+ title="SE – Swedish Certification Body for IT Security FMV/CSEC" height="32"
+ width="32" style="margin-bottom:-1em" /><br />SE</a>
+ </td>
+ </tr>
+
+ <tr class="even">
+ <td class="b">
+ <!-- ID: 2019.1265 -->
+ Magic SSO V4.0<a name="2019.1265" style="text-decoration:none;"> </a><span
+ title="2019.1265"> </span>
+ <div class="none">
+ <!-- <a href="https://www.dreamsecurity.com/" title="Vendor's web site" target="_blank">Dreamsecurity Co., Ltd.</a> -->
+ <img src="/assets/images/spacer.gif" width="600" height="1" class="" align="top"> <br />
+ <!-- ------ ------ ------ Document Links ------ ------ ------ -->
+ <div style="line-height:2.5em">
+ <a href="/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf"
+ title="Certification Report: KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf"
+ target="_blank" class="button2">Certification Report</a>
+ <a href="/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf"
+ title="Security Target: Magic_SSO_V4.0-ST-v1.4_EN.pdf" target="_blank"
+ class="button2">Security Target</a>
+
+ <!-- how it was before 2017.09
+ <a href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf" target="_blank" class="button2" title="PP: Korean National Protection Profile for Single Sign On V1.0">Protection Profile</a> -->
+ <!-- ------------------------------------ -->
+ <!-- Set the display text for the link to the PP -->
+
+ <!-- ------------------------------------ -->
+ <br /><a
+ href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf"
+ target="_blank" class="button2" title="">Korean National Protection Profile for Single
+ Sign On V1.0</a>
+
+ </div>
+ <!-- ------ ------ ------ END Document Links ------ ------ ------ -->
+ <!-- ------ ------ ------ Product Updates ------ ------ ------ -->
+
+ <!-- ------ ------ ------ END Product Updates ------ ------ ------ -->
+ </div>
+ </td>
+ <!--end-product-cell-->
+ <td>
+ <a href="https://www.dreamsecurity.com/" title="Vendor's web site"
+ target="_blank">Dreamsecurity Co., Ltd.</a>
+ </td>
+ <td>
+
+
+
+ </td>
+ <td style="text-align:center"><span title="2019.1265">2019-11-15</span></td>
+
+ <td style="text-align:center">2024-11-15</td>
+
+
+ <td>
+
+
+ PP Compliant
+ </td>
+
+ <td style="text-align:center">
+ <a href="/ccra/members/#KR">
+ <img src="/assets/images/flags/kr.png"
+ alt="KR – IT Security Certification Center(ITSCC)"
+ title="KR – IT Security Certification Center(ITSCC)" height="32"
+ width="32" style="margin-bottom:-1em" /><br />KR</a>
+ </td>
+ </tr>
+
+ </tbody>
+ </table>
+
+
+ </div><!-- END white, i think -->
+
+
+
+
+
+ <!-- 2017.07.10 - Added blurb to active list re. archiving in 2019//KC -->
+ <!-- 2017.09.06 - 2017.09.08 - Big cosmetic changes per NIAP's request//KC -->
+ <!-- 2017.09.06 - Code clean-up in areas I touched//KC -->
+ <!-- 2017.09.08 - Converted page to one paired CFoutput tag//KC -->
+ <!-- 2018.04.03 - 2018.04.05 - Put compliance column back w a few changes//KC -->
+ <!-- 2018.10.04 - Added new text//KC -->
+ <!-- 2019.12.16 - Replaced Application.WebRoot w Application.CodeBase//KC -->
+ <!-- 2019.12.19 - Added product ID to comment & span title//KC -->
+
+
+ <div id="footer" class="copy">
+ <p> </p>
+
+ </div>
+
+
+ <script type="text/javascript" src="/assets/js/searchhi.js"></script>
+
+
+
+
+ <!-- jstablesorter sppt -->
+ <!-- zxcv - Should be w/in JSTableSorter -->
+
+ <script type="text/javascript">
+ $(document).ready(function () {
+ $.tablesorter.defaults.widgets = ['zebra']; // add stripes
+ $.tablesorter.defaults.sortList = [[3, 1]]; // default sort column(s)
+ $('table').tablesorter({ headers: { 5: { sorter: false } } }); // enable sort & disable sort on specific column(s)
+ });
+ </script>
+ <!-- END jstablesorter sppt -->
+
+
+ <!--[if lt IE 8]>
+<script src="/assets/js/ie7-js/IE8.js" type="text/javascript"></script>
+<![endif]-->
+
+
+ <!-- Google Analytics -->
+ <script type="text/javascript">
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-1394658-2']);
+ _gaq.push(['_trackPageview']);
+ (function () {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+ </script>
+
+
+
+
+
+ <!-- 2017.09.07 - Added jstablesorter sppt//KC -->
+
+
+ </div>
+ </div>
+</body>
+
+</html>
\ No newline at end of file diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json new file mode 100644 index 00000000..44a422db --- /dev/null +++ b/test/data/test_cc_oop/fictional_cert.json @@ -0,0 +1,30 @@ +{ + "category": "Sample category", + "name": "Sample certificate name", + "manufacturer": "Sample manufacturer", + "scheme": "Sample scheme", + "security_level": [ + "Sample security level" + ], + "not_valid_before": "1900-01-02", + "not_valid_after": "1900-01-03", + "report_link": "http://path.to/report/link", + "st_link": "http://path.to/st/link", + "src": "custom", + "cert_link": "http://path.to/cert/link", + "manufacturer_web": "http://path.to/manufacturer/web", + "protection_profiles": [ + { + "pp_name": "sample_pp", + "pp_link": "http://sample.pp" + } + ], + "maintainance_updates": [ + { + "maintainance_date": "1900-01-01", + "maintainance_title": "Sample maintainance", + "maintainance_report_link": "https://maintainance.up", + "maintainance_st_link": "https://maintainance.up" + } + ] +}
\ No newline at end of file diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json new file mode 100644 index 00000000..49d612b5 --- /dev/null +++ b/test/data/test_cc_oop/toy_dataset.json @@ -0,0 +1,50 @@ +{ + "root_dir": "/fictional/path/to/dataset", + "timestamp": "2020-11-16 17:04:14.770153", + "sha256_digest": "not implemented", + "name": "toy dataset", + "description": "toy dataset description", + "n_certs": 2, + "certs": [ + { + "category": "Access Control Devices and Systems", + "name": "NetIQ Identity Manager 4.7", + "manufacturer": "NetIQ Corporation", + "scheme": "SE", + "security_level": [ + "ALC_FLR.2", + "EAL3+" + ], + "not_valid_before": "2020-06-15", + "not_valid_after": "2025-06-15", + "report_link": "http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf", + "st_link": "http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf", + "src": "csv + html", + "cert_link": "http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf", + "manufacturer_web": "https://www.netiq.com/", + "protection_profiles": [], + "maintainance_updates": [] + }, + { + "category": "Access Control Devices and Systems", + "name": "Magic SSO V4.0", + "manufacturer": "Dreamsecurity Co., Ltd.", + "scheme": "KR", + "security_level": [], + "not_valid_before": "2019-11-15", + "not_valid_after": "2024-11-15", + "report_link": "http://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf", + "st_link": "http://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf", + "src": "csv + html", + "cert_link": null, + "manufacturer_web": "https://www.dreamsecurity.com/", + "protection_profiles": [ + { + "pp_name": "Korean National Protection Profile for Single Sign On V1.0", + "pp_link": "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf" + } + ], + "maintainance_updates": [] + } + ] +}
\ No newline at end of file diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py new file mode 100644 index 00000000..c29e90e3 --- /dev/null +++ b/test/test_cc_oop.py @@ -0,0 +1,127 @@ +from unittest import TestCase +from pathlib import Path +from tempfile import TemporaryDirectory, mkstemp +from datetime import date, datetime +import json +import filecmp +import shutil +import os + +from sec_certs.dataset import CCDataset, DatasetJSONDecoder, DatasetJSONEncoder +from sec_certs.certificate import CommonCriteriaCert + + +class TestCommonCriteriaOOP(TestCase): + def setUp(self): + self.test_data_dir = Path(__file__).parent / 'data' / 'test_cc_oop' + self.crt_one = CommonCriteriaCert('Access Control Devices and Systems', + 'NetIQ Identity Manager 4.7', + 'NetIQ Corporation', + 'SE', + {'ALC_FLR.2', + 'EAL3+'}, + date(2020, 6, 15), + date(2025, 6, 15), + 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf', + 'http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf', + 'csv + html', + 'http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf', + 'https://www.netiq.com/', + set(), + set()) + + self.crt_two = CommonCriteriaCert('Access Control Devices and Systems', + 'Magic SSO V4.0', + 'Dreamsecurity Co., Ltd.', + 'KR', + set(), + date(2019, 11, 15), + date(2024, 11, 15), + 'http://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf', + 'http://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf', + 'csv + html', + None, + 'https://www.dreamsecurity.com/', + {CommonCriteriaCert.ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0', + 'http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf')}, + set()) + + pp = CommonCriteriaCert.ProtectionProfile('sample_pp', 'http://sample.pp') + update = CommonCriteriaCert.MaintainanceReport(date(1900, 1, 1), 'Sample maintainance', 'https://maintainance.up', 'https://maintainance.up') + self.fictional_cert = CommonCriteriaCert('Sample category', + 'Sample certificate name', + 'Sample manufacturer', + 'Sample scheme', + {'Sample security level'}, + date(1900, 1, 2), + date(1900, 1, 3), + 'http://path.to/report/link', + 'http://path.to/st/link', + 'custom', + 'http://path.to/cert/link', + 'http://path.to/manufacturer/web', + {pp}, + {update}) + self.template_dataset = CCDataset({self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, Path('/fictional/path/to/dataset'), 'toy dataset', 'toy dataset description') + self.template_dataset.timestamp = datetime(2020, 11, 16, hour=17, minute=4, second=14, microsecond=770153) + + def test_certificate_input_sanity(self): + self.assertEqual(self.crt_one.report_link, + 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf', + 'Report link contains some improperly escaped characters.') + + @staticmethod + def equal_to_json(referential_path, obj): + fd, path = mkstemp() + try: + with os.fdopen(fd, 'w') as handle: + json.dump(obj, handle, cls=DatasetJSONEncoder, indent=4) + + return filecmp.cmp(referential_path, path) + finally: + os.remove(path) + + @staticmethod + def equal_from_json(referential_path, obj): + with open(referential_path, 'r') as handle: + new_obj = json.load(handle, cls=DatasetJSONDecoder) + return obj == new_obj + + def test_cert_to_json(self): + self.assertTrue(self.equal_to_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), + 'The certificate serialized to json differs from a template.') + + def test_dataset_to_json(self): + self.assertTrue(self.equal_to_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset), + 'The dataset serialized to json differs from a template.') + + def test_cert_from_json(self): + self.assertTrue(self.equal_from_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), + 'The certificate serialized from json differs from a template.') + + def test_dataset_from_json(self): + self.assertTrue(self.equal_from_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset), + 'The dataset serialized from json differs from a template.') + + def test_build_empty_dataset(self): + with TemporaryDirectory() as tmp_dir: + dset = CCDataset({}, Path(tmp_dir), 'sample_dataset', 'sample dataset description') + dset.get_certs_from_web(to_download=False, get_archived=False, get_active=False) + self.assertEqual(len(dset), 0, 'The dataset should contain 0 files.') + + def test_build_dataset(self): + with TemporaryDirectory() as tmp_dir: + dataset_path = Path(tmp_dir) + os.mkdir(dataset_path / 'web') + shutil.copyfile(self.test_data_dir / 'cc_products_active.csv', dataset_path / 'web' / 'cc_products_active.csv') + shutil.copyfile(self.test_data_dir / 'cc_products_active.html', dataset_path / 'web' / 'cc_products_active.html') + + dset = CCDataset({}, dataset_path, 'sample_dataset', 'sample dataset description') + dset.get_certs_from_web(keep_metadata=False, to_download=False, get_archived=False, get_active=True) + + self.assertEqual(len(os.listdir(dataset_path)), 0, + 'Meta files (csv, html) were not deleted properly albeit this was explicitly required.') + + self.assertEqual(len(dset), 2, 'The dataset should contain 2 files.') + self.assertTrue(self.crt_one in dset, 'The dataset does not contain the template certificate.') + self.assertEqual(dset, self.template_dataset, 'The loaded dataset does not match the template dataset.') |