From e265d2545e49c640b1b2e97745f19fa34cf816ed Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Tue, 24 Nov 2020 13:49:21 +0100 Subject: adds download CC cert_reports and sec_target Allow for download of pdfs, specifically certificate_reports and security_targets in a parallel way. --- sec_certs/constants.py | 2 ++ sec_certs/dataset.py | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/sec_certs/constants.py b/sec_certs/constants.py index 3ffb867c..b88729e1 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -1,5 +1,7 @@ from enum import Enum +N_THREADS = 8 +RESPONSE_OK = 200 class CertFramework(Enum): CC = 'Common Criteria' diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 6f0912cd..ec08f532 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -9,6 +9,7 @@ from importlib import import_module from abc import ABC, abstractmethod from pathlib import Path import shutil +from multiprocessing import Pool from tabula import read_pdf import pandas as pd @@ -20,6 +21,8 @@ from sec_certs.helpers import find_tables, repair_pdf from sec_certs.certificate import CommonCriteriaCert, Certificate, FIPSCertificate from sec_certs.extract_certificates import extract_certificates_keywords from sec_certs.constants import FIPS_NOT_AVAILABLE_CERT_SIZE +import sec_certs.constants as constants +import sec_certs.download as download class Dataset(ABC): @@ -65,7 +68,7 @@ class Dataset(ABC): @classmethod def from_dict(cls, dct: Dict): certs = {x.dgst: x for x in dct['certs']} - return cls(certs, dct['root_dir'], dct['name'], dct['description']) + return cls(certs, Path(dct['root_dir']), dct['name'], dct['description']) @classmethod def from_csv(cls): @@ -100,6 +103,18 @@ class CCDataset(Dataset): def web_dir(self) -> Path: return self.root_dir / 'web' + @property + def certs_dir(self) -> Path: + return self.root_dir / 'certs' + + @property + def reports_dir(self) -> Path: + return self.certs_dir / 'reports' + + @property + def targets_dir(self) -> Path: + return self.certs_dir / 'targets' + html_products = { 'cc_products_active.html': 'https://www.commoncriteriaportal.org/products/', 'cc_products_archived.html': 'https://www.commoncriteriaportal.org/products/index.cfm?archived=1', @@ -299,6 +314,28 @@ class CCDataset(Dataset): return certs + def download_pdfs(self, urls, paths): + responses = download.download_parallel(list(zip(urls, paths)), constants.N_THREADS) + for r in responses: + if r[1] != constants.RESPONSE_OK: + logging.warning(f'Receieved response: {r[1]} when downloading {r[0]}') + + def download_reports(self): + self.reports_dir.mkdir(parents=True, exist_ok=True) + reports_urls = [x.report_link for x in self] + paths = [self.reports_dir / (x.dgst + '.pdf') for x in self] + self.download_pdfs(reports_urls, paths) + + def download_targets(self): + self.targets_dir.mkdir(parents=True, exist_ok=True) + target_urls = [x.st_link for x in self] + paths = [self.targets_dir / (x.dgst + '.pdf') for x in self] + self.download_pdfs(target_urls, paths) + + def download_all_pdfs(self): + self.download_reports() + self.download_targets() + class FIPSDataset(Dataset): FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov' -- cgit v1.3.1 From e0a714d0ecec6b5b342b3bde4bba1732b8ede2e4 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Wed, 25 Nov 2020 14:32:21 +0100 Subject: Alpha version of cert download and convert - Adds the ability to download pdf certificates in a dataset. - introduces local path variable into dataset object. - Ability to convert pdf certificates into txt certificates. --- sec_certs/constants.py | 4 +++ sec_certs/dataset.py | 97 ++++++++++++++++++++++++++++++++++++++++++-------- sec_certs/download.py | 4 +++ sec_certs/helpers.py | 9 +++++ 4 files changed, 99 insertions(+), 15 deletions(-) diff --git a/sec_certs/constants.py b/sec_certs/constants.py index b88729e1..560cad17 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -2,6 +2,10 @@ from enum import Enum N_THREADS = 8 RESPONSE_OK = 200 +RETURNCODE_OK = 0 + +MIN_CORRECT_CERT_SIZE = 5000 + class CertFramework(Enum): CC = 'Common Criteria' diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 80acc87a..9414b510 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -1,9 +1,9 @@ import os import re -from datetime import datetime +from datetime import datetime, time import locale import logging -from typing import Dict, List, ClassVar +from typing import Dict, List, ClassVar, Collection import json from importlib import import_module @@ -11,7 +11,10 @@ import copy from abc import ABC, abstractmethod from pathlib import Path import shutil -from multiprocessing import Pool +from multiprocessing import Pool, pool +import tqdm +from functools import partial + from tabula import read_pdf import pandas as pd @@ -100,8 +103,25 @@ class Dataset(ABC): logging.info( f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.') + @staticmethod + def convert_pdfs_to_text(pdf_paths: Collection[Path], txt_paths: Collection[Path]): + assert len(pdf_paths) == len(txt_paths) + results = [] + partial_convert_pdf = partial(helpers.convert_pdf_file, options=['-raw']) + with tqdm.tqdm(total=len(pdf_paths)) as progress: + for result in pool.ThreadPool(constants.N_THREADS).imap(partial_convert_pdf, zip(pdf_paths, txt_paths)): + progress.update(1) + results.append(result) + + @staticmethod + def get_corrupted_pdfs(pdf_paths): + return [p for p in pdf_paths if p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE] + class CCDataset(Dataset): + def __init__(self, certs, root_dir, name, description): + super().__init__(certs, root_dir, name, description) + @property def web_dir(self) -> Path: return self.root_dir / 'web' @@ -114,10 +134,42 @@ class CCDataset(Dataset): def reports_dir(self) -> Path: return self.certs_dir / 'reports' + @property + def reports_pdf_dir(self) -> Path: + return self.reports_dir / 'pdf' + + @property + def reports_txt_dir(self) -> Path: + return self.reports_dir / 'txt' + @property def targets_dir(self) -> Path: return self.certs_dir / 'targets' + @property + def targets_pdf_dir(self) -> Path: + return self.targets_dir / 'pdf' + + @property + def targets_txt_dir(self) -> Path: + return self.targets_dir / 'txt' + + @property + def report_pdf_paths(self) -> Dict[str, Path]: + return {x: self.reports_pdf_dir / (self[x].dgst + '.pdf') for x in self.certs} + + @property + def report_txt_paths(self) -> Dict[str, Path]: + return {x: self.reports_txt_dir / (self[x].dgst + '.txt') for x in self.certs} + + @property + def target_pdf_paths(self) -> Dict[str, Path]: + return {x: self.targets_pdf_dir / (self[x].dgst + '.pdf') for x in self.certs} + + @property + def target_txt_paths(self) -> Dict[str, Path]: + return {x: self.targets_txt_dir / (self[x].dgst + '.txt') for x in self.certs} + html_products = { 'cc_products_active.html': 'https://www.commoncriteriaportal.org/products/', 'cc_products_archived.html': 'https://www.commoncriteriaportal.org/products/index.cfm?archived=1', @@ -330,28 +382,43 @@ class CCDataset(Dataset): return certs - def download_pdfs(self, urls, paths): - responses = download.download_parallel(list(zip(urls, paths)), constants.N_THREADS) - for r in responses: - if r[1] != constants.RESPONSE_OK: - logging.warning(f'Receieved response: {r[1]} when downloading {r[0]}') - def download_reports(self): - self.reports_dir.mkdir(parents=True, exist_ok=True) + self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) reports_urls = [x.report_link for x in self] - paths = [self.reports_dir / (x.dgst + '.pdf') for x in self] - self.download_pdfs(reports_urls, paths) + download.download_parallel(list(zip(reports_urls, self.report_pdf_paths.values())), constants.N_THREADS) def download_targets(self): - self.targets_dir.mkdir(parents=True, exist_ok=True) + self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) target_urls = [x.st_link for x in self] - paths = [self.targets_dir / (x.dgst + '.pdf') for x in self] - self.download_pdfs(target_urls, paths) + download.download_parallel(list(zip(target_urls, self.target_pdf_paths.values())), constants.N_THREADS) def download_all_pdfs(self): + logging.info('Downloading CC certificate reports') self.download_reports() + + # TODO: Do checks below live when downloading and re-download straight away? + corrupted_reports = self.get_corrupted_pdfs(self.report_pdf_paths.values()) + for r in corrupted_reports: + logging.error(f'Corrupted pdf file at: {r}') + + logging.info('Downloading CC security targets') self.download_targets() + # TODO: Do checks below live when downloading and re-download straight away? + corrupted_targets = self.get_corrupted_pdfs(self.target_pdf_paths.values()) + for t in corrupted_targets: + logging.error(f'Corrupted pdf file at: {t}') + + def convert_all_pdfs(self): + # TODO: Get rid of the list() invocation here. + logging.info('Converting CC certificate reports to .txt') + self.reports_txt_dir.mkdir(parents=True, exist_ok=True) + self.convert_pdfs_to_text(list(self.report_pdf_paths.values()), list(self.report_txt_paths.values())) + + logging.info('Converting CC security targets to .txt') + self.targets_txt_dir.mkdir(parents=True, exist_ok=True) + self.convert_pdfs_to_text(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values())) + class FIPSDataset(Dataset): FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov' diff --git a/sec_certs/download.py b/sec_certs/download.py index 16ac91de..d01da79f 100644 --- a/sec_certs/download.py +++ b/sec_certs/download.py @@ -1,4 +1,5 @@ import os +import logging from multiprocessing.pool import ThreadPool from pathlib import Path from tqdm import tqdm @@ -7,6 +8,7 @@ from typing import Sequence, Tuple, List import requests from .files import search_files +import sec_certs.constants as constants CC_WEB_URL = 'https://www.commoncriteriaportal.org' @@ -29,6 +31,8 @@ def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Se for response in pool.imap(download, items): progress.update(1) responses.append(response) + if response[1] != constants.RESPONSE_OK: + logging.error(f'Request for url {response[0]} returned {response[1]}') pool.close() pool.join() return responses diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index dd95b0c8..7db64f55 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -12,8 +12,11 @@ from typing import Union from datetime import date import numpy as np import pandas as pd +import subprocess from bs4 import Tag, NavigableString +import sec_certs.constants as constants + def download_file(url: str, output: Path) -> int: r = requests.get(url, allow_redirects=True) @@ -147,6 +150,12 @@ def repair_pdf(file: Path): pdf.save(file) +def convert_pdf_file(filepaths: Tuple[Path, Path], options): + pdf_path, txt_path = filepaths[0], filepaths[1] + proc_result = subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if proc_result.returncode != constants.RETURNCODE_OK: + logging.error(f'Converting pdf {pdf_path} resulted into the following result: {proc_result}') + return proc_result -- cgit v1.3.1 From 991edee4af994c664aa9a00bc848136c1c162255 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Wed, 25 Nov 2020 15:13:03 +0100 Subject: renamed oop_demo to cc_oop_demo --- cc_oop_demo.py | 35 +++++++++++++++++++++++++++++++++++ oop_demo.py | 35 ----------------------------------- 2 files changed, 35 insertions(+), 35 deletions(-) create mode 100644 cc_oop_demo.py delete mode 100644 oop_demo.py diff --git a/cc_oop_demo.py b/cc_oop_demo.py new file mode 100644 index 00000000..f8ae5508 --- /dev/null +++ b/cc_oop_demo.py @@ -0,0 +1,35 @@ +from sec_certs.dataset import CCDataset +from sec_certs.serialization import CustomJSONEncoder, CustomJSONDecoder +from pathlib import Path +from datetime import datetime +import logging +import json + + +def main(): + logging.basicConfig(level=logging.INFO) + start = datetime.now() + + # Create empty dataset + dset = CCDataset({}, Path('./debug_dataset'), 'sample_dataset', 'sample dataset description') + + # Load metadata for certificates from CSV and HTML sources + dset.get_certs_from_web() + logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') + + # Dump dataset into JSON + with open('./debug_dataset/cc_full_dataset.json', 'w') as handle: + json.dump(dset, handle, cls=CustomJSONEncoder, indent=4) + + # Load dataset from JSON + with open('./debug_dataset/cc_full_dataset.json', 'r') as handle: + new_dset = json.load(handle, cls=CustomJSONDecoder) + + assert dset == new_dset + + end = datetime.now() + logging.info(f'The computation took {(end-start)} seconds.') + + +if __name__ == '__main__': + main() diff --git a/oop_demo.py b/oop_demo.py deleted file mode 100644 index f8ae5508..00000000 --- a/oop_demo.py +++ /dev/null @@ -1,35 +0,0 @@ -from sec_certs.dataset import CCDataset -from sec_certs.serialization import CustomJSONEncoder, CustomJSONDecoder -from pathlib import Path -from datetime import datetime -import logging -import json - - -def main(): - logging.basicConfig(level=logging.INFO) - start = datetime.now() - - # Create empty dataset - dset = CCDataset({}, Path('./debug_dataset'), 'sample_dataset', 'sample dataset description') - - # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web() - logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') - - # Dump dataset into JSON - with open('./debug_dataset/cc_full_dataset.json', 'w') as handle: - json.dump(dset, handle, cls=CustomJSONEncoder, indent=4) - - # Load dataset from JSON - with open('./debug_dataset/cc_full_dataset.json', 'r') as handle: - new_dset = json.load(handle, cls=CustomJSONDecoder) - - assert dset == new_dset - - end = datetime.now() - logging.info(f'The computation took {(end-start)} seconds.') - - -if __name__ == '__main__': - main() -- cgit v1.3.1 From 6fe316a44e9313d83a779c87d53bc14b1001b9c0 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Wed, 25 Nov 2020 15:17:26 +0100 Subject: add dataset print method --- sec_certs/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 9414b510..8535fd42 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -57,7 +57,7 @@ class Dataset(ABC): return self.certs == other.certs def __str__(self) -> str: - return 'Not implemented' + return str(type(self).__name__) + ':' + self.name + ', ' + str(len(self)) + ' certificates' def to_csv(self): pass -- cgit v1.3.1 From a0c8ce2bd6c27b805ab037f3637322c1324709a0 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Wed, 25 Nov 2020 15:19:01 +0100 Subject: sanity check on dataset size on deserialization --- sec_certs/dataset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 8535fd42..efa2740b 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -73,7 +73,9 @@ class Dataset(ABC): @classmethod def from_dict(cls, dct: Dict): certs = {x.dgst: x for x in dct['certs']} - return cls(certs, Path(dct['root_dir']), dct['name'], dct['description']) + dset = cls(certs, Path(dct['root_dir']), dct['name'], dct['description']) + assert len(dset) == dct['n_certs'] + return dset @classmethod def from_csv(cls): -- cgit v1.3.1 From d4cf3bf8b2329181d0f5bf17c3dd87564fa80b2a Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Wed, 25 Nov 2020 15:46:59 +0100 Subject: Test for download of pdf files The test assumes that the files download correctly. --- sec_certs/helpers.py | 8 ++++++++ test/test_cc_oop.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 7db64f55..718b66cd 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -45,6 +45,14 @@ def get_first_16_bytes_sha256(string: str) -> str: return hashlib.sha256(string.encode('utf-8')).hexdigest()[:16] +def get_sha256_filepath(filepath): + hash_sha256 = hashlib.sha256() + with open(filepath, "rb") as f: + for chunk in iter(lambda: f.read(4096), b''): + hash_sha256.update(chunk) + return hash_sha256.hexdigest() + + def sanitize_link(record: str) -> Union[str, None]: if not record: return None diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py index b5740077..3a965ba7 100644 --- a/test/test_cc_oop.py +++ b/test/test_cc_oop.py @@ -10,6 +10,7 @@ import os from sec_certs.dataset import CCDataset from sec_certs.serialization import CustomJSONEncoder, CustomJSONDecoder from sec_certs.certificate import CommonCriteriaCert +import sec_certs.helpers as helpers class TestCommonCriteriaOOP(TestCase): @@ -66,6 +67,11 @@ class TestCommonCriteriaOOP(TestCase): self.template_dataset = CCDataset({self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, Path('/fictional/path/to/dataset'), 'toy dataset', 'toy dataset description') self.template_dataset.timestamp = datetime(2020, 11, 16, hour=17, minute=4, second=14, microsecond=770153) + self.template_report_pdf_hashes = {'869415cc4b91282e': '774c41fbba980191ca40ae610b2f61484c5997417b3325b6fd68b345173bde52', + '2d010ecfb604747a': '533a5995ef8b736cc48cfda30e8aafec77d285511471e0e5a9e8007c8750203a'} + self.template_target_pdf_hashes = {'869415cc4b91282e': 'b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9', + '2d010ecfb604747a': '3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be'} + def test_certificate_input_sanity(self): self.assertEqual(self.crt_one.report_link, 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf', @@ -88,6 +94,22 @@ class TestCommonCriteriaOOP(TestCase): new_obj = json.load(handle, cls=CustomJSONDecoder) return obj == new_obj + def test_download_pdfs(self): + with open(self.test_data_dir / 'toy_dataset.json', 'r') as handle: + dset = json.load(handle, cls=CustomJSONDecoder) + + with TemporaryDirectory() as td: + dset.root_dir = Path(td) + + dset.download_all_pdfs() + dset.convert_all_pdfs() + + actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.report_pdf_paths.items()} + actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.target_pdf_paths.items()} + + self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (certificate report) do not the template') + self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template') + def test_cert_to_json(self): self.assertTrue(self.equal_to_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), 'The certificate serialized to json differs from a template.') -- cgit v1.3.1 From 2332b94b57c90ea280d2b62e1c5f776ff11f18b9 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 26 Nov 2020 08:35:56 +0100 Subject: get rid of convert_pdf() in tests --- test/test_cc_oop.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py index 3a965ba7..f6471821 100644 --- a/test/test_cc_oop.py +++ b/test/test_cc_oop.py @@ -102,7 +102,6 @@ class TestCommonCriteriaOOP(TestCase): dset.root_dir = Path(td) dset.download_all_pdfs() - dset.convert_all_pdfs() actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.report_pdf_paths.items()} actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.target_pdf_paths.items()} -- cgit v1.3.1 From 56093596cd176a45bbf210e9e4def967cf3d0141 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 26 Nov 2020 13:55:49 +0100 Subject: Finalize proposal of cert download and convert The proposed method of processing certificates is breadth-first. By that I mean that single task is done for all certificates, then next task is evaluated. All tasks should be implemented as static methods and are to be paralelized by calling `sec-certs/cert_processing.py`. Currently, a generic method `process_paralel()` sits there that is capable of pretty much arbitrary parallel processing (both using threads and processes). Any methods to-be called by this method must require unpacked arguments on the input. --- cc_oop_demo.py | 6 ++++ sec_certs/cert_processing.py | 28 +++++++++++++++ sec_certs/dataset.py | 84 +++++++++++++++++++++++++++----------------- sec_certs/download.py | 6 +--- sec_certs/helpers.py | 24 ++++++------- 5 files changed, 97 insertions(+), 51 deletions(-) create mode 100644 sec_certs/cert_processing.py diff --git a/cc_oop_demo.py b/cc_oop_demo.py index f8ae5508..fae5d911 100644 --- a/cc_oop_demo.py +++ b/cc_oop_demo.py @@ -27,6 +27,12 @@ def main(): assert dset == new_dset + # Download pdfs + dset.download_all_pdfs() + + # Convert pdfs to text + dset.convert_all_pdfs() + end = datetime.now() logging.info(f'The computation took {(end-start)} seconds.') diff --git a/sec_certs/cert_processing.py b/sec_certs/cert_processing.py new file mode 100644 index 00000000..6d0f4d92 --- /dev/null +++ b/sec_certs/cert_processing.py @@ -0,0 +1,28 @@ +from tqdm import tqdm +from multiprocessing.pool import Pool, ThreadPool +import sec_certs.constants as constants +import logging +import time + +# TODO: Add timeout. Kinda meh with ThreadingTimeout, SignalTimeout does not work on Windows, stopit package. +def process_parallel(func, items, max_workers, callback=None, use_threading=True, progress_bar=True): + if use_threading is True: + pool = ThreadPool(max_workers) + else: + pool = Pool(max_workers) + + results = [pool.apply_async(func, (*i, ), callback=callback) for i in items] + + if progress_bar is True: + bar = tqdm(total=len(results)) + while not all([x.ready() for x in results]): + done_count = len([x.ready() for x in results if x.ready()]) + bar.update(done_count - bar.n) + time.sleep(1) + bar.update(len(results) - bar.n) + bar.close() + + pool.close() + pool.join() + + return [r.get() for r in results] \ No newline at end of file diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index efa2740b..703aab88 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -1,6 +1,6 @@ import os import re -from datetime import datetime, time +from datetime import datetime import locale import logging from typing import Dict, List, ClassVar, Collection @@ -11,9 +11,9 @@ import copy from abc import ABC, abstractmethod from pathlib import Path import shutil -from multiprocessing import Pool, pool -import tqdm + from functools import partial +import requests from tabula import read_pdf @@ -28,6 +28,7 @@ from sec_certs.extract_certificates import extract_certificates_keywords from sec_certs.constants import FIPS_NOT_AVAILABLE_CERT_SIZE import sec_certs.constants as constants import sec_certs.download as download +import sec_certs.cert_processing as cert_processing class Dataset(ABC): @@ -73,7 +74,7 @@ class Dataset(ABC): @classmethod def from_dict(cls, dct: Dict): certs = {x.dgst: x for x in dct['certs']} - dset = cls(certs, Path(dct['root_dir']), dct['name'], dct['description']) + dset = cls(certs, Path(dct['root_dir']), dct['name'], dct['description']) assert len(dset) == dct['n_certs'] return dset @@ -106,24 +107,41 @@ class Dataset(ABC): f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.') @staticmethod - def convert_pdfs_to_text(pdf_paths: Collection[Path], txt_paths: Collection[Path]): + def convert_pdfs_to_txt(pdf_paths: Collection[Path], txt_paths: Collection[Path]): assert len(pdf_paths) == len(txt_paths) - results = [] + partial_convert_pdf = partial(helpers.convert_pdf_file, options=['-raw']) - with tqdm.tqdm(total=len(pdf_paths)) as progress: - for result in pool.ThreadPool(constants.N_THREADS).imap(partial_convert_pdf, zip(pdf_paths, txt_paths)): - progress.update(1) - results.append(result) + exit_codes = cert_processing.process_parallel(partial_convert_pdf, + list(zip(pdf_paths, txt_paths)), + constants.N_THREADS) + + n_successful = len([e for e in exit_codes if e == constants.RETURNCODE_OK]) + logging.info(f'Successfully converted {n_successful} files pdf->txt, {len(exit_codes) - n_successful} failed.') + + for path, e in zip(pdf_paths, exit_codes): + if e != constants.RETURNCODE_OK: + logging.info(f'Failed to convert {path}, exit code: {e}') @staticmethod - def get_corrupted_pdfs(pdf_paths): - return [p for p in pdf_paths if p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE] + def download_parallel(urls, paths, prune_corrupted=True): + exit_codes = cert_processing.process_parallel(download.download_file, + list(zip(urls, paths)), + constants.N_THREADS) + n_successful = len([e for e in exit_codes if e == requests.codes.ok]) + logging.info(f'Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.') + for url, e in zip(urls, exit_codes): + if e != requests.codes.ok: + logging.error(f'Failed to download {url}, exit code: {e}') + + if prune_corrupted is True: + for p in paths: + if p.exists() and p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE: + logging.error(f'Corrupted file at: {p}') + # TODO: Delete -class CCDataset(Dataset): - def __init__(self, certs, root_dir, name, description): - super().__init__(certs, root_dir, name, description) +class CCDataset(Dataset): @property def web_dir(self) -> Path: return self.root_dir / 'web' @@ -210,10 +228,13 @@ class CCDataset(Dataset): html_items = [x for x in html_items if 'archived' not in str(x[1])] csv_items = [x for x in csv_items if 'archived' not in str(x[1])] + html_urls, html_paths = [x[0] for x in html_items], [x[1] for x in html_items] + csv_urls, csv_paths = [x[0] for x in csv_items], [x[1] for x in csv_items] + if to_download is True: logging.info('Downloading required csv and html files.') - helpers.download_parallel(html_items, num_threads=8) - helpers.download_parallel(csv_items, num_threads=8) + self.download_parallel(html_urls, html_paths) + self.download_parallel(csv_urls, csv_paths) logging.info('Adding CSV certificates to CommonCriteria dataset.') csv_certs = self.get_all_certs_from_csv(get_active, get_archived) @@ -387,39 +408,36 @@ class CCDataset(Dataset): def download_reports(self): self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) reports_urls = [x.report_link for x in self] - download.download_parallel(list(zip(reports_urls, self.report_pdf_paths.values())), constants.N_THREADS) + self.download_parallel(reports_urls, self.report_pdf_paths.values(), prune_corrupted=True) def download_targets(self): self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) target_urls = [x.st_link for x in self] - download.download_parallel(list(zip(target_urls, self.target_pdf_paths.values())), constants.N_THREADS) + self.download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) def download_all_pdfs(self): logging.info('Downloading CC certificate reports') self.download_reports() - # TODO: Do checks below live when downloading and re-download straight away? - corrupted_reports = self.get_corrupted_pdfs(self.report_pdf_paths.values()) - for r in corrupted_reports: - logging.error(f'Corrupted pdf file at: {r}') - logging.info('Downloading CC security targets') self.download_targets() - # TODO: Do checks below live when downloading and re-download straight away? - corrupted_targets = self.get_corrupted_pdfs(self.target_pdf_paths.values()) - for t in corrupted_targets: - logging.error(f'Corrupted pdf file at: {t}') + def convert_reports_to_txt(self): + self.reports_txt_dir.mkdir(parents=True, exist_ok=True) + # TODO: Get rid of the list() invocation here. + self.convert_pdfs_to_txt(list(self.report_pdf_paths.values()), list(self.report_txt_paths.values())) - def convert_all_pdfs(self): + def convert_targets_to_txt(self): + self.targets_txt_dir.mkdir(parents=True, exist_ok=True) # TODO: Get rid of the list() invocation here. + self.convert_pdfs_to_txt(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values())) + + def convert_all_pdfs(self): logging.info('Converting CC certificate reports to .txt') - self.reports_txt_dir.mkdir(parents=True, exist_ok=True) - self.convert_pdfs_to_text(list(self.report_pdf_paths.values()), list(self.report_txt_paths.values())) + self.convert_reports_to_txt() logging.info('Converting CC security targets to .txt') - self.targets_txt_dir.mkdir(parents=True, exist_ok=True) - self.convert_pdfs_to_text(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values())) + self.convert_targets_to_txt() class FIPSDataset(Dataset): diff --git a/sec_certs/download.py b/sec_certs/download.py index d01da79f..fc225150 100644 --- a/sec_certs/download.py +++ b/sec_certs/download.py @@ -1,5 +1,4 @@ import os -import logging from multiprocessing.pool import ThreadPool from pathlib import Path from tqdm import tqdm @@ -7,8 +6,7 @@ from typing import Sequence, Tuple, List import requests -from .files import search_files -import sec_certs.constants as constants +from sec_certs.files import search_files CC_WEB_URL = 'https://www.commoncriteriaportal.org' @@ -31,8 +29,6 @@ def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Se for response in pool.imap(download, items): progress.update(1) responses.append(response) - if response[1] != constants.RESPONSE_OK: - logging.error(f'Request for url {response[0]} returned {response[1]}') pool.close() pool.join() return responses diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 718b66cd..942a37fd 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -13,16 +13,17 @@ from datetime import date import numpy as np import pandas as pd import subprocess -from bs4 import Tag, NavigableString - -import sec_certs.constants as constants def download_file(url: str, output: Path) -> int: - r = requests.get(url, allow_redirects=True) - with output.open('wb') as f: - f.write(r.content) - return r.status_code + try: + r = requests.get(url, allow_redirects=True, timeout=5) + if r.status_code == requests.codes.ok: + with output.open("wb") as f: + f.write(r.content) + return r.status_code + except requests.exceptions.Timeout: + return requests.codes.timeout def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Sequence[Tuple[str, int]]: @@ -158,12 +159,9 @@ def repair_pdf(file: Path): pdf.save(file) -def convert_pdf_file(filepaths: Tuple[Path, Path], options): - pdf_path, txt_path = filepaths[0], filepaths[1] - proc_result = subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - if proc_result.returncode != constants.RETURNCODE_OK: - logging.error(f'Converting pdf {pdf_path} resulted into the following result: {proc_result}') - return proc_result +def convert_pdf_file(pdf_path: Path, txt_path: Path, options): + return subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode + -- cgit v1.3.1 From cad8f53ea493cd16f5a4d539d24884b3c8d18213 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 26 Nov 2020 15:40:10 +0100 Subject: move merge_certs() to CCDataset --- sec_certs/cert_processing.py | 2 -- sec_certs/dataset.py | 49 ++++++++++++++++++++++++-------------------- sec_certs/helpers.py | 7 +------ 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/sec_certs/cert_processing.py b/sec_certs/cert_processing.py index 6d0f4d92..bc11083b 100644 --- a/sec_certs/cert_processing.py +++ b/sec_certs/cert_processing.py @@ -1,7 +1,5 @@ from tqdm import tqdm from multiprocessing.pool import Pool, ThreadPool -import sec_certs.constants as constants -import logging import time # TODO: Add timeout. Kinda meh with ThreadingTimeout, SignalTimeout does not work on Windows, stopit package. diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 703aab88..6aed4f91 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -3,7 +3,7 @@ import re from datetime import datetime import locale import logging -from typing import Dict, List, ClassVar, Collection +from typing import Dict, List, ClassVar, Collection, TypeVar, Type import json from importlib import import_module @@ -15,11 +15,11 @@ import shutil from functools import partial import requests - from tabula import read_pdf import pandas as pd from bs4 import BeautifulSoup + from sec_certs.files import search_files from sec_certs import helpers as helpers from sec_certs.helpers import find_tables, repair_pdf @@ -32,7 +32,7 @@ import sec_certs.cert_processing as cert_processing class Dataset(ABC): - def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name', + def __init__(self, certs: Dict[str, 'Certificate'], root_dir: Path, name: str = 'dataset name', description: str = 'dataset_description'): self.root_dir = root_dir self.timestamp = datetime.now() @@ -89,23 +89,6 @@ class Dataset(ABC): def get_certs_from_web(self): pass - def merge_certs(self, certs: Dict[str, 'Certificate']): - """ - Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates - """ - will_be_added = {} - n_merged = 0 - for crt in certs.values(): - if crt not in self: - will_be_added[crt.dgst] = crt - else: - self[crt.dgst].merge(crt) - n_merged += 1 - - self.certs.update(will_be_added) - logging.info( - f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.') - @staticmethod def convert_pdfs_to_txt(pdf_paths: Collection[Path], txt_paths: Collection[Path]): assert len(pdf_paths) == len(txt_paths) @@ -113,7 +96,8 @@ class Dataset(ABC): partial_convert_pdf = partial(helpers.convert_pdf_file, options=['-raw']) exit_codes = cert_processing.process_parallel(partial_convert_pdf, list(zip(pdf_paths, txt_paths)), - constants.N_THREADS) + constants.N_THREADS, + use_threading=False) n_successful = len([e for e in exit_codes if e == constants.RETURNCODE_OK]) logging.info(f'Successfully converted {n_successful} files pdf->txt, {len(exit_codes) - n_successful} failed.') @@ -142,6 +126,10 @@ class Dataset(ABC): class CCDataset(Dataset): + def __init__(self, certs: Dict[str, 'CommonCriteriaCert'], root_dir: Path, name: str = 'dataset name', + description: str = 'dataset_description'): + super().__init__(certs, root_dir, name, description) + @property def web_dir(self) -> Path: return self.root_dir / 'web' @@ -209,6 +197,23 @@ class CCDataset(Dataset): 'cc_pp_archived.csv': 'https://www.commoncriteriaportal.org/pps/pps-archived.csv' } + def merge_certs(self, certs: Dict[str, 'CommonCriteriaCert']): + """ + Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates + """ + will_be_added = {} + n_merged = 0 + for crt in certs.values(): + if crt not in self: + will_be_added[crt.dgst] = crt + else: + self[crt.dgst].merge(crt) + n_merged += 1 + + self.certs.update(will_be_added) + logging.info( + f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.') + def get_certs_from_web(self, to_download=True, keep_metadata: bool = True, get_active=True, get_archived=True): """ Downloads all metadata about certificates from CSV and HTML sources @@ -396,7 +401,7 @@ class CCDataset(Dataset): ] cat_dict = {x: y for (x, y) in zip(cc_table_ids, cc_categories)} - with open(file, 'r') as handle: + with file.open('r') as handle: soup = BeautifulSoup(handle, 'html.parser') certs = {} diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 942a37fd..44538f28 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -160,9 +160,4 @@ def repair_pdf(file: Path): def convert_pdf_file(pdf_path: Path, txt_path: Path, options): - return subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode - - - - - + return subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60).returncode -- cgit v1.3.1 From 0afb2ed33761186da942fee64b8500d6490c2eef Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 26 Nov 2020 16:07:50 +0100 Subject: replace 403 st links with None --- sec_certs/certificate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index f5fb3f83..76db21ac 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -323,6 +323,7 @@ class FIPSCertificate(Certificate): class CommonCriteriaCert(Certificate): cc_url = 'http://www.commoncriteriaportal.org' + empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/' @dataclass(eq=True, frozen=True) class MaintainanceReport: @@ -398,6 +399,9 @@ class CommonCriteriaCert(Certificate): self.protection_profiles = protection_profiles self.maintainance_updates = maintainance_updates + if self.st_link == self.empty_st_url: + self.st_link = None + @property def dgst(self) -> str: """ -- cgit v1.3.1 From dcfad971a66ef34a88e3bb16942e0c01d3125510 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 Nov 2020 09:44:20 +0100 Subject: added private/public methods to dataset,certificate --- sec_certs/certificate.py | 48 +++++++++++++------------- sec_certs/dataset.py | 87 +++++++++++++++++++++++------------------------- 2 files changed, 65 insertions(+), 70 deletions(-) diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index 76db21ac..375ba029 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -449,28 +449,28 @@ class CommonCriteriaCert(Certificate): Creates a CC certificate from html row """ - def get_name(cell: Tag) -> str: + def _get_name(cell: Tag) -> str: return list(cell.stripped_strings)[0] - def get_manufacturer(cell: Tag) -> Optional[str]: + def _get_manufacturer(cell: Tag) -> Optional[str]: if lst := list(cell.stripped_strings): return lst[0] else: return None - def get_scheme(cell: Tag) -> str: + def _get_scheme(cell: Tag) -> str: return list(cell.stripped_strings)[0] - def get_security_level(cell: Tag) -> set: + def _get_security_level(cell: Tag) -> set: return set(cell.stripped_strings) - def get_manufacturer_web(cell: Tag) -> Optional[str]: + def _get_manufacturer_web(cell: Tag) -> Optional[str]: for link in cell.find_all('a'): if link is not None and link.get('title') == 'Vendor\'s web site' and link.get('href') != 'http://': return link.get('href') return None - def get_protection_profiles(cell: Tag) -> set: + def _get_protection_profiles(cell: Tag) -> set: protection_profiles = set() for link in list(cell.find_all('a')): if link.get('href') is not None and '/ppfiles/' in link.get('href'): @@ -479,13 +479,13 @@ class CommonCriteriaCert(Certificate): 'href'))) return protection_profiles - def get_date(cell: Tag) -> date: + def _get_date(cell: Tag) -> date: text = cell.get_text() extracted_date = datetime.strptime( text, '%Y-%m-%d').date() if text else None return extracted_date - def get_report_st_links(cell: Tag) -> (str, str): + def _get_report_st_links(cell: Tag) -> (str, str): links = cell.find_all('a') # TODO: Exception checks assert links[1].get('title').startswith('Certification Report') @@ -497,18 +497,18 @@ class CommonCriteriaCert(Certificate): return report_link, security_target_link - def get_cert_link(cell: Tag) -> Optional[str]: + def _get_cert_link(cell: Tag) -> Optional[str]: links = cell.find_all('a') return CommonCriteriaCert.cc_url + links[0].get('href') if links else None - def get_maintainance_div(cell: Tag) -> Optional[Tag]: + def _get_maintainance_div(cell: Tag) -> Optional[Tag]: divs = cell.find_all('div') for d in divs: if d.find('div') and d.stripped_strings and list(d.stripped_strings)[0] == 'Maintenance Report(s)': return d return None - def get_maintainance_updates(main_div: Tag) -> set: + def _get_maintainance_updates(main_div: Tag) -> set: possible_updates = list(main_div.find_all('li')) maintainance_updates = set() for u in possible_updates: @@ -537,19 +537,19 @@ class CommonCriteriaCert(Certificate): logging.error('Unexpected number of cells in CC html row.') raise - name = get_name(cells[0]) - manufacturer = get_manufacturer(cells[1]) - manufacturer_web = get_manufacturer_web(cells[1]) - scheme = get_scheme(cells[6]) - security_level = get_security_level(cells[5]) - protection_profiles = get_protection_profiles(cells[0]) - not_valid_before = get_date(cells[3]) - not_valid_after = get_date(cells[4]) - report_link, st_link = get_report_st_links(cells[0]) - cert_link = get_cert_link(cells[2]) - - maintainance_div = get_maintainance_div(cells[0]) - maintainances = get_maintainance_updates( + name = _get_name(cells[0]) + manufacturer = _get_manufacturer(cells[1]) + manufacturer_web = _get_manufacturer_web(cells[1]) + scheme = _get_scheme(cells[6]) + security_level = _get_security_level(cells[5]) + protection_profiles = _get_protection_profiles(cells[0]) + not_valid_before = _get_date(cells[3]) + not_valid_after = _get_date(cells[4]) + report_link, st_link = _get_report_st_links(cells[0]) + cert_link = _get_cert_link(cells[2]) + + maintainance_div = _get_maintainance_div(cells[0]) + maintainances = _get_maintainance_updates( maintainance_div) if maintainance_div else set() return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 6aed4f91..db8f6dc1 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -60,12 +60,6 @@ class Dataset(ABC): def __str__(self) -> str: return str(type(self).__name__) + ':' + self.name + ', ' + str(len(self)) + ' certificates' - def to_csv(self): - pass - - def to_dataframe(self): - pass - def to_dict(self): return {'root_dir': copy.deepcopy(self.root_dir), 'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest, 'name': self.name, 'description': self.description, @@ -78,19 +72,20 @@ class Dataset(ABC): assert len(dset) == dct['n_certs'] return dset - @classmethod - def from_csv(cls): - pass + @abstractmethod + def get_certs_from_web(self): + raise NotImplementedError('Not meant to be implemented by the base class.') - def dump_to_json(self): - pass + @abstractmethod + def convert_all_pdfs(self): + raise NotImplementedError('Not meant to be implemented by the base class.') @abstractmethod - def get_certs_from_web(self): - pass + def download_all_pdfs(self): + raise NotImplementedError('Not meant to be implemented by the base class.') @staticmethod - def convert_pdfs_to_txt(pdf_paths: Collection[Path], txt_paths: Collection[Path]): + def _convert_pdfs_to_txt(pdf_paths: Collection[Path], txt_paths: Collection[Path]): assert len(pdf_paths) == len(txt_paths) partial_convert_pdf = partial(helpers.convert_pdf_file, options=['-raw']) @@ -107,7 +102,7 @@ class Dataset(ABC): logging.info(f'Failed to convert {path}, exit code: {e}') @staticmethod - def download_parallel(urls, paths, prune_corrupted=True): + def _download_parallel(urls, paths, prune_corrupted=True): exit_codes = cert_processing.process_parallel(download.download_file, list(zip(urls, paths)), constants.N_THREADS) @@ -197,7 +192,7 @@ class CCDataset(Dataset): 'cc_pp_archived.csv': 'https://www.commoncriteriaportal.org/pps/pps-archived.csv' } - def merge_certs(self, certs: Dict[str, 'CommonCriteriaCert']): + def _merge_certs(self, certs: Dict[str, 'CommonCriteriaCert']): """ Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates """ @@ -238,24 +233,24 @@ class CCDataset(Dataset): if to_download is True: logging.info('Downloading required csv and html files.') - self.download_parallel(html_urls, html_paths) - self.download_parallel(csv_urls, csv_paths) + self._download_parallel(html_urls, html_paths) + self._download_parallel(csv_urls, csv_paths) logging.info('Adding CSV certificates to CommonCriteria dataset.') - csv_certs = self.get_all_certs_from_csv(get_active, get_archived) - self.merge_certs(csv_certs) + csv_certs = self._get_all_certs_from_csv(get_active, get_archived) + self._merge_certs(csv_certs) # TODO: Someway along the way, 3 certificates get lost. Investigate and fix. logging.info('Adding HTML certificates to CommonCriteria dataset.') - html_certs = self.get_all_certs_from_html(get_active, get_archived) - self.merge_certs(html_certs) + html_certs = self._get_all_certs_from_html(get_active, get_archived) + self._merge_certs(html_certs) logging.info(f'The resulting dataset has {len(self)} certificates.') if not keep_metadata: shutil.rmtree(self.web_dir) - def get_all_certs_from_csv(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: + def _get_all_certs_from_csv(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: """ Creates dictionary of new certificates from csv sources. """ @@ -267,19 +262,19 @@ class CCDataset(Dataset): new_certs = {} for file in csv_sources: - partial_certs = self.parse_single_csv(self.web_dir / file) + partial_certs = self._parse_single_csv(self.web_dir / file) logging.info( f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) return new_certs @staticmethod - def parse_single_csv(file: Path) -> Dict[str, 'CommonCriteriaCert']: + def _parse_single_csv(file: Path) -> Dict[str, 'CommonCriteriaCert']: """ Using pandas, this parses a single CSV file. """ - def get_primary_key_str(row): + def _get_primary_key_str(row): prim_key = row['category'] + row['cert_name'] + row['report_link'] return prim_key @@ -298,7 +293,7 @@ class CCDataset(Dataset): ['not_valid_before', 'not_valid_after', 'maintainance_date']].apply(pd.to_datetime) df['dgst'] = df.apply(lambda row: helpers.get_first_16_bytes_sha256( - get_primary_key_str(row)), axis=1) + _get_primary_key_str(row)), axis=1) df_base = df.loc[df.is_maintainance == False].copy() df_main = df.loc[df.is_maintainance == True].copy() @@ -326,7 +321,7 @@ class CCDataset(Dataset): df_base.itertuples()} return certs - def get_all_certs_from_html(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: + def _get_all_certs_from_html(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: """ Prepares dictionary of certificates from all html files. """ @@ -338,19 +333,19 @@ class CCDataset(Dataset): new_certs = {} for file in html_sources: - partial_certs = self.parse_single_html(self.web_dir / file) + partial_certs = self._parse_single_html(self.web_dir / file) logging.info( f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) return new_certs @staticmethod - def parse_single_html(file: Path) -> Dict[str, 'CommonCriteriaCert']: + def _parse_single_html(file: Path) -> Dict[str, 'CommonCriteriaCert']: """ Prepares a dictionary of certificates from a single html file. """ - def get_timestamp_from_footer(footer): + def _get_timestamp_from_footer(footer): locale.setlocale(locale.LC_ALL, 'en_US') footer_text = list(footer.stripped_strings)[0] date_string = footer_text.split(',')[1:3] @@ -359,7 +354,7 @@ class CCDataset(Dataset): date_string[1] + ' ' + time_string return datetime.strptime(formatted_datetime, ' %B %d %Y %I:%M %p') - def parse_table(soup: BeautifulSoup, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']: + def _parse_table(soup: BeautifulSoup, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']: tables = soup.find_all('table', id=table_id) assert len(tables) <= 1 @@ -371,7 +366,7 @@ class CCDataset(Dataset): header, footer, body = rows[0], rows[1], rows[2:] # TODO: It's possible to obtain timestamp of the moment when the list was generated. It's identical for each table and should thus only be obtained once. Not necessarily in each table - # timestamp = get_timestamp_from_footer(footer) + # timestamp = _get_timestamp_from_footer(footer) # TODO: Do we have use for number of expected certs? We get rid of duplicites, so no use for assert expected == actual # caption_str = str(table.findAll('caption')) @@ -406,43 +401,43 @@ class CCDataset(Dataset): certs = {} for key, val in cat_dict.items(): - certs.update(parse_table(soup, key, val)) + certs.update(_parse_table(soup, key, val)) return certs - def download_reports(self): + def _download_reports(self): self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) reports_urls = [x.report_link for x in self] - self.download_parallel(reports_urls, self.report_pdf_paths.values(), prune_corrupted=True) + self._download_parallel(reports_urls, self.report_pdf_paths.values(), prune_corrupted=True) - def download_targets(self): + def _download_targets(self): self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) target_urls = [x.st_link for x in self] - self.download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) + self._download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) def download_all_pdfs(self): logging.info('Downloading CC certificate reports') - self.download_reports() + self._download_reports() logging.info('Downloading CC security targets') - self.download_targets() + self._download_targets() - def convert_reports_to_txt(self): + def _convert_reports_to_txt(self): self.reports_txt_dir.mkdir(parents=True, exist_ok=True) # TODO: Get rid of the list() invocation here. - self.convert_pdfs_to_txt(list(self.report_pdf_paths.values()), list(self.report_txt_paths.values())) + self._convert_pdfs_to_txt(list(self.report_pdf_paths.values()), list(self.report_txt_paths.values())) - def convert_targets_to_txt(self): + def _convert_targets_to_txt(self): self.targets_txt_dir.mkdir(parents=True, exist_ok=True) # TODO: Get rid of the list() invocation here. - self.convert_pdfs_to_txt(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values())) + self._convert_pdfs_to_txt(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values())) def convert_all_pdfs(self): logging.info('Converting CC certificate reports to .txt') - self.convert_reports_to_txt() + self._convert_reports_to_txt() logging.info('Converting CC security targets to .txt') - self.convert_targets_to_txt() + self._convert_targets_to_txt() class FIPSDataset(Dataset): -- cgit v1.3.1 From 02d9f746e0223ee88338d53d9cc346ba268c448c Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 Nov 2020 10:36:34 +0100 Subject: Implement better logging - Now logging on info level, each module into its owner logger. - Logging is performed on standard output - Logging is performed into file `./cert_processing_log.txt` that is gitignored --- .gitignore | 3 +++ cc_oop_demo.py | 17 +++++++++++---- sec_certs/certificate.py | 16 +++++++------- sec_certs/constants.py | 1 + sec_certs/dataset.py | 54 +++++++++++++++++++++++++----------------------- sec_certs/helpers.py | 5 +++-- 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index ea3366db..38b49e31 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ venv.bak/ # mypy .mypy_cache/ + +# log +./cert_processing_log.txt \ No newline at end of file diff --git a/cc_oop_demo.py b/cc_oop_demo.py index fae5d911..54e07307 100644 --- a/cc_oop_demo.py +++ b/cc_oop_demo.py @@ -1,13 +1,21 @@ from sec_certs.dataset import CCDataset from sec_certs.serialization import CustomJSONEncoder, CustomJSONDecoder +import sec_certs.constants as constants from pathlib import Path from datetime import datetime import logging import json +logger = logging.getLogger(__name__) + def main(): - logging.basicConfig(level=logging.INFO) + file_handler = logging.FileHandler(constants.LOGS_FILENAME) + stream_handler = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + file_handler.setFormatter(formatter) + stream_handler.setFormatter(formatter) + logging.basicConfig(level=logging.INFO, handlers=[file_handler, stream_handler]) start = datetime.now() # Create empty dataset @@ -15,7 +23,7 @@ def main(): # Load metadata for certificates from CSV and HTML sources dset.get_certs_from_web() - logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') + logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') # Dump dataset into JSON with open('./debug_dataset/cc_full_dataset.json', 'w') as handle: @@ -24,6 +32,7 @@ def main(): # Load dataset from JSON with open('./debug_dataset/cc_full_dataset.json', 'r') as handle: new_dset = json.load(handle, cls=CustomJSONDecoder) + new_dset.root_dir = Path('/Users/adam/phd/projects/certificates/sec-certs/debug_dataset') assert dset == new_dset @@ -31,10 +40,10 @@ def main(): dset.download_all_pdfs() # Convert pdfs to text - dset.convert_all_pdfs() + new_dset.convert_all_pdfs() end = datetime.now() - logging.info(f'The computation took {(end-start)} seconds.') + logger.info(f'The computation took {(end-start)} seconds.') if __name__ == '__main__': diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index 375ba029..d2585188 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -12,6 +12,8 @@ from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type from sec_certs import helpers, extract_certificates +logger = logging.getLogger(__name__) + class Certificate(ABC): T = TypeVar('T', bound='Certificate') @@ -225,7 +227,7 @@ class FIPSCertificate(Certificate): html_items_found['fips_vendor'] = vendor_string if html_items_found['fips_vendor'] == '': - logging.warning(f"WARNING: NO VENDOR FOUND{current_file}") + logger.warning(f"WARNING: NO VENDOR FOUND{current_file}") @staticmethod def parse_lab(current_div: Tag, html_items_found: Dict, current_file: Path): @@ -236,10 +238,10 @@ class FIPSCertificate(Certificate): 'div', 'panel-body').children)[2].strip().split('\n')[1].strip() if html_items_found['fips_lab'] == '': - logging.warning(f"WARNING: NO LAB FOUND{current_file}") + loggerr.warning(f"WARNING: NO LAB FOUND{current_file}") if html_items_found['fips_nvlap_code'] == '': - logging.warning(f"WARNING: NO NVLAP CODE FOUND{current_file}") + logger.warning(f"WARNING: NO NVLAP CODE FOUND{current_file}") @staticmethod def parse_related_files(current_div: Tag, html_items_found: Dict): @@ -416,7 +418,7 @@ class CommonCriteriaCert(Certificate): On other values (apart from maintainances, see TODO below) the sanity checks are made. """ if self != other: - logging.warning( + logger.warning( f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}') for att, val in vars(self).items(): @@ -431,7 +433,7 @@ class CommonCriteriaCert(Certificate): pass # This is expected else: if getattr(self, att) != getattr(other, att): - logging.warning( + logger.warning( f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}') if self.src != other.src: self.src = self.src + ' + ' + other.src @@ -527,14 +529,14 @@ class CommonCriteriaCert(Certificate): main_st_link = CommonCriteriaCert.cc_url + \ l.get('href') else: - logging.error('Unknown link in Maintenance part!') + logger.error('Unknown link in Maintenance part!') maintainance_updates.add( CommonCriteriaCert.MaintainanceReport(main_date, main_title, main_report_link, main_st_link)) return maintainance_updates cells = list(row.find_all('td')) if len(cells) != 7: - logging.error('Unexpected number of cells in CC html row.') + logger.error('Unexpected number of cells in CC html row.') raise name = _get_name(cells[0]) diff --git a/sec_certs/constants.py b/sec_certs/constants.py index 560cad17..c3bf646e 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -6,6 +6,7 @@ RETURNCODE_OK = 0 MIN_CORRECT_CERT_SIZE = 5000 +LOGS_FILENAME = './cert_processing_log.txt' class CertFramework(Enum): CC = 'Common Criteria' diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index db8f6dc1..0bfc7fb0 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -30,6 +30,8 @@ import sec_certs.constants as constants import sec_certs.download as download import sec_certs.cert_processing as cert_processing +logger = logging.getLogger(__name__) + class Dataset(ABC): def __init__(self, certs: Dict[str, 'Certificate'], root_dir: Path, name: str = 'dataset name', @@ -95,11 +97,11 @@ class Dataset(ABC): use_threading=False) n_successful = len([e for e in exit_codes if e == constants.RETURNCODE_OK]) - logging.info(f'Successfully converted {n_successful} files pdf->txt, {len(exit_codes) - n_successful} failed.') + logger.info(f'Successfully converted {n_successful} files pdf->txt, {len(exit_codes) - n_successful} failed.') for path, e in zip(pdf_paths, exit_codes): if e != constants.RETURNCODE_OK: - logging.info(f'Failed to convert {path}, exit code: {e}') + logger.info(f'Failed to convert {path}, exit code: {e}') @staticmethod def _download_parallel(urls, paths, prune_corrupted=True): @@ -107,16 +109,16 @@ class Dataset(ABC): list(zip(urls, paths)), constants.N_THREADS) n_successful = len([e for e in exit_codes if e == requests.codes.ok]) - logging.info(f'Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.') + logger.info(f'Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.') for url, e in zip(urls, exit_codes): if e != requests.codes.ok: - logging.error(f'Failed to download {url}, exit code: {e}') + logger.error(f'Failed to download {url}, exit code: {e}') if prune_corrupted is True: for p in paths: if p.exists() and p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE: - logging.error(f'Corrupted file at: {p}') + logger.error(f'Corrupted file at: {p}') # TODO: Delete @@ -206,7 +208,7 @@ class CCDataset(Dataset): n_merged += 1 self.certs.update(will_be_added) - logging.info( + logger.info( f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.') def get_certs_from_web(self, to_download=True, keep_metadata: bool = True, get_active=True, get_archived=True): @@ -232,20 +234,20 @@ class CCDataset(Dataset): csv_urls, csv_paths = [x[0] for x in csv_items], [x[1] for x in csv_items] if to_download is True: - logging.info('Downloading required csv and html files.') + logger.info('Downloading required csv and html files.') self._download_parallel(html_urls, html_paths) self._download_parallel(csv_urls, csv_paths) - logging.info('Adding CSV certificates to CommonCriteria dataset.') + logger.info('Adding CSV certificates to CommonCriteria dataset.') csv_certs = self._get_all_certs_from_csv(get_active, get_archived) self._merge_certs(csv_certs) # TODO: Someway along the way, 3 certificates get lost. Investigate and fix. - logging.info('Adding HTML certificates to CommonCriteria dataset.') + logger.info('Adding HTML certificates to CommonCriteria dataset.') html_certs = self._get_all_certs_from_html(get_active, get_archived) self._merge_certs(html_certs) - logging.info(f'The resulting dataset has {len(self)} certificates.') + logger.info(f'The resulting dataset has {len(self)} certificates.') if not keep_metadata: shutil.rmtree(self.web_dir) @@ -263,7 +265,7 @@ class CCDataset(Dataset): new_certs = {} for file in csv_sources: partial_certs = self._parse_single_csv(self.web_dir / file) - logging.info( + logger.info( f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) return new_certs @@ -300,7 +302,7 @@ class CCDataset(Dataset): n_all = len(df_base) n_deduplicated = len(df_base.drop_duplicates(subset=['dgst'])) if (n_dup := n_all - n_deduplicated) > 0: - logging.warning( + logger.warning( f'The CSV {file} contains {n_dup} duplicates by the primary key.') df_base = df_base.drop_duplicates(subset=['dgst']) @@ -334,7 +336,7 @@ class CCDataset(Dataset): new_certs = {} for file in html_sources: partial_certs = self._parse_single_html(self.web_dir / file) - logging.info( + logger.info( f'Parsed {len(partial_certs)} certificates from: {file}') new_certs.update(partial_certs) return new_certs @@ -416,10 +418,10 @@ class CCDataset(Dataset): self._download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) def download_all_pdfs(self): - logging.info('Downloading CC certificate reports') + logger.info('Downloading CC certificate reports') self._download_reports() - logging.info('Downloading CC security targets') + logger.info('Downloading CC security targets') self._download_targets() def _convert_reports_to_txt(self): @@ -433,10 +435,10 @@ class CCDataset(Dataset): self._convert_pdfs_to_txt(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values())) def convert_all_pdfs(self): - logging.info('Converting CC certificate reports to .txt') + logger.info('Converting CC certificate reports to .txt') self._convert_reports_to_txt() - logging.info('Converting CC security targets to .txt') + logger.info('Converting CC security targets to .txt') self._convert_targets_to_txt() @@ -501,7 +503,7 @@ class FIPSDataset(Dataset): def get_certs_from_web(self): def get_certificates_from_html(html_file: Path) -> None: - logging.info(f'Getting certificate ids from {html_file}') + logger.info(f'Getting certificate ids from {html_file}') html = BeautifulSoup(open(html_file).read(), 'html.parser') table = [x for x in html.find( @@ -509,7 +511,7 @@ class FIPSDataset(Dataset): for entry in table: self.certs[entry.find('a').text] = {} - logging.info("Downloading required html files") + logger.info("Downloading required html files") self.web_dir.mkdir(parents=True, exist_ok=True) self.policies_dir.mkdir(exist_ok=True) @@ -531,7 +533,7 @@ class FIPSDataset(Dataset): for f in html_files: get_certificates_from_html(self.web_dir / f) - logging.info('Downloading certficate html and security policies') + logger.info('Downloading certficate html and security policies') html_items = [ (f"https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{cert_id}", self.web_dir / f"{cert_id}.html") for cert_id in list(self.certs.keys()) if @@ -544,7 +546,7 @@ class FIPSDataset(Dataset): _, self.new_files = helpers.download_parallel( html_items + sp_items, 8), len(html_items) + len(sp_items) - logging.info(f"{self.new_files} needed to be downloaded") + logger.info(f"{self.new_files} needed to be downloaded") if self.new_files > 0 or not (self.root_dir / 'fips_full_dataset.json').exists(): # if False: @@ -552,7 +554,7 @@ class FIPSDataset(Dataset): self.certs[cert] = FIPSCertificate.html_from_file( self.web_dir / f'{cert}.html') else: - logging.info("Certs loaded from previous scanning") + logger.info("Certs loaded from previous scanning") dataset = json.loads(open(self.root_dir / 'fips_full_dataset.json').read(), cls=import_module('sec_certs.serialization').CustomJSONDecoder) self.certs = dataset.certs @@ -657,10 +659,10 @@ class FIPSDataset(Dataset): self.certs[file_name].file_status = False break if broken_files: - logging.warning("CERTIFICATE FILES WITH WRONG CERTIFICATES PARSED") - logging.warning(broken_files) - logging.warning("... skipping these...") - logging.warning(f"Total non-analyzable files:{len(broken_files)}") + logger.warning("CERTIFICATE FILES WITH WRONG CERTIFICATES PARSED") + logger.warning(broken_files) + logger.warning("... skipping these...") + logger.warning(f"Total non-analyzable files:{len(broken_files)}") for file_name in self.keywords: self.certs[file_name].connections = [] diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 44538f28..5219a897 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -14,6 +14,7 @@ import numpy as np import pandas as pd import subprocess +logger = logging.getLogger(__name__) def download_file(url: str, output: Path) -> int: try: @@ -120,7 +121,7 @@ def find_tables_iterative(file_text: str) -> List[int]: if line.startswith('Table ') or line.startswith('Exhibit'): pages.add(current_page) if not pages: - logging.warning('No pages found') + logger.warning('No pages found') return list(pages) @@ -143,7 +144,7 @@ def find_tables(txt: str, file_name: Path) -> Optional[List]: return None # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer - logging.info(f'parsing tables in {file_name}') + logger.info(f'parsing tables in {file_name}') rb = find_tables_iterative(txt) return rb if rb else None -- cgit v1.3.1 From 0ffa845fbb51f6ac1f7b4434b3a78cc4d7f38d22 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 Nov 2020 11:03:59 +0100 Subject: added test for CC pdf conversion --- .travis.yml | 3 + test/data/test_cc_oop/report_869415cc4b91282e.txt | 481 +++++++ test/data/test_cc_oop/target_869415cc4b91282e.txt | 1497 +++++++++++++++++++++ test/test_cc_oop.py | 12 +- 4 files changed, 1992 insertions(+), 1 deletion(-) create mode 100644 test/data/test_cc_oop/report_869415cc4b91282e.txt create mode 100644 test/data/test_cc_oop/target_869415cc4b91282e.txt diff --git a/.travis.yml b/.travis.yml index 21416ba2..e67232e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ language: python dist: xenial python: "3.8" +before_install: + - sudo apt-get -y install poppler-utils + install: - pip install ".[dev,test]" diff --git a/test/data/test_cc_oop/report_869415cc4b91282e.txt b/test/data/test_cc_oop/report_869415cc4b91282e.txt new file mode 100644 index 00000000..0f421a31 --- /dev/null +++ b/test/data/test_cc_oop/report_869415cc4b91282e.txt @@ -0,0 +1,481 @@ +Ärendetyp: 6 Diarienummer: 18FMV7705-43:1 +HEMLIG/ +enligt Offentlighets- och sekretesslagen +(2009:400) +2020-06-15 +Country of origin: Sweden +Försvarets materielverk +Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +Issue: 1.0, 2020-Jun-15 +Authorisation: Helén Svensson, Lead Certifier , CSEC + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +2 (18) +Table of Contents +1 Executive Summary 3 +2 Identification 5 +3 Security Policy 6 +3.1 Security Management 6 +3.2 Security Audit 6 +3.3 Identification and Authentication 6 +3.4 User Data Protection 7 +3.5 Trusted Path / Channel 7 +3.6 Cryptographic Support 7 +4 Assumptions and Clarification of Scope 8 +4.1 Usage Assumptions 8 +4.2 Environmental Assumptions 8 +4.3 Clarification of Scope 8 +5 Architectural Information 9 +6 Documentation 11 +7 IT Product Testing 12 +7.1 Developer Testing 12 +7.2 Evaluator Testing 12 +7.3 Penetration Testing 12 +8 Evaluated Configuration 13 +9 Results of the Evaluation 14 +10 Evaluator Comments and Recommendations 15 +11 Glossary 16 +12 Bibliography 17 +Appendix A Scheme Versions 18 +A.1 Scheme/Quality Management System 18 +A.2 Scheme Notes 18 + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +3 (18) +1 Executive Summary +The TOE is NetIQ Identity Manager 4.7. +It is a software TOE consisting of the components listed below that can be setup on +separate hardware platforms, see the [ST], or as a virtual appliances. +TOE Components: + Identity Applications (RBPM) 4.7.3.0.1109 + Identity Manager Engine 4.7.3.0.AE + Identity Reporting Module 6.5.0. F14508F + Sentinel Log Management for Identity Governance and Administration +8.2.2.0_5415 + One SSO Provider (OSP) 6.3.3.0 + Self Service Password Reset (SSPR) 4.4.0.2 B366 r39762 +The TOE is delivered as software with documentation and can be installed in a physi- +cal or virtual environment. +It is important to verify the integrity of the TOE for secure acceptance of the TOE in +accordance with the preparative procedures of the guidance, i.e. verify the TLS con- +nection, the CA certificate and the file hash. It is also important to update the TOE (in- +cluding 3rd party software) and the operational environment of the TOE in accordance +with the preparative procedures of the guidance to mitigate known vulnerabilities. +No conformance claims to any PP are made for the TOE. +The evaluation has been performed by Combitech AB in Växjö, Sweden and by +EWA-Canada in Ottawa, Canada. Site Visit and parts of the testing was performed at +the developer's site in Bangalore, India. +The evaluation was completed on 2020-06-02. The evaluation was conducted in ac- +cordance with the requirements of Common Criteria, version 3.1 R5. +Combitech AB is a licensed evaluation facility for Common Criteria under the Swe- +dish Common Criteria Evaluation and Certification Scheme. Combitech AB is also +accredited by the Swedish accreditation body SWEDAC according to ISO/IEC 17025 +for Common Criteria evaluation. EWA-Canada Ltd. operates as a Foreign location for +Combitech AB within scope of the Swedish Common Criteria Evaluation and Certifi- +cation Scheme. +The certifier monitored the activities of the evaluator by reviewing all successive ver- +sions of the evaluation reports, and by observing site-visit and testing. The certifier +determined that the evaluation results confirm the security claims in the Security +Target (ST) and the Common Methodology for evaluation assurance level EAL3 +augmented by ALC_FLR.2 +The certification results only apply to the version of the product indicated in the +certificate, and on the condition that all the stipulations in the Security Target are +met. +This certificate is not an endorsement of the IT product by CSEC or any other or- +ganisation that recognises or gives effect to this certificate, and no warranty of the +IT product by CSEC or any other organisation that recognises or gives effect to this +certificate is either expressed or implied. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +4 (18) +As specified in the security target of this evaluation, the invocation of cryptographic +primitives has been included in the TOE, while the implementation of these primi- +tives has been located in TOE environment. Therefore the invocation of crypto- +graphic primitives has been in the scope of this evaluation, while correctness of im- +plementation of cryptographic primitives been excluded from the TOE. Correctness +of implementation is done through third party certification Cryptographic Module +Validation Program (CMVP) certificate number 1747 referred to in the Security +Target. +Users of this product are advised to consider their acceptance of this third party af- +firmation regarding the correctness of implementation of the cryptographic primi- +tives. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +5 (18) +2 Identification +Certification Identification +Certification ID CSEC2018013 +Name and version of the cer- +tified IT product +NetIQ® Identity Manager 4.7 +TOE components: + Identity Applications (RBPM) 4.7.3.0.1109 + Identity Manager Engine 4.7.3.0.AE + Identity Reporting Module 6.5.0. F14508F + Sentinel Log Management for Identity Govern- +ance and Administration 8.2.2.0_5415 + One SSO Provider (OSP) 6.3.3.0 + Self Service Password Reset (SSPR) 4.4.0.2 +B366 r39762 +Security Target Identification NetIQ Identity Manager 4.7 Security Target (ST), +NetIQ Corporation , 2020-06-01, document version +2.6 +EAL EAL3 + ALC_FLR.2 +Sponsor NetIQ Corporation +Developer NetIQ Corporation +ITSEF Combitech AB and EWA-Canada +Common Criteria version 3.1 release 5 +CEM version 3.1 release 5 +QMS version 1.23.2 +Scheme Notes Release 15.0 +Recognition Scope CCRA, SOGIS and EA/MLA +Certification date 2020-06-15 + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +6 (18) +3 Security Policy +The security features performed by the TOE are as follows: + Security Management + Security Audit + Identification and Authentication + User Data Protection + Trusted Path / Channels + Cryptographic Support +3.1 Security Management +The TOE maintains operator roles. The individual roles are categorized into two main +roles: the Administrator and the User. +Administrator - A user who has rights to configure and manage all aspects of the TOE +User - The user's capabilities can be configured to: + View hierarchical relationships between User objects + View and edit user information (with appropriate rights). + Search for users or resources using advanced search criteria (which can be saved +for later reuse). + Recover forgotten passwords. +Only an Administrator can determine the behavior of, disable, enable, and modify the +behavior of the functions that implement the Discretionary Access Control SFP. The +TPE ensures only secure values are accepted for the security attributes listed with Dis- +cretionary Access Control SFP. +3.2 Security Audit +The TOE generates the following audit data: + Start-up and shutdown of the audit functions (instantiated by startup of the TOE) + User login/logout + Login failures +The TOE provides the Administrator with the capability to read all audit data gener- +ated within the TOE via the console. The GUI provides a suitable means for an Ad- +ministrator to interpret the information from the audit log. +The A.TIMESOURCE is added to the assumptions on operational environment, and +OE.TIME is added to the operational environment security objectives. The time and +date provided by the operational environment are used to form the timestamps. The +TOE ensures that the audit trail data is stamped when recorded with a dependable date +and time received from the OE (operating system). In this manner, accurate time and +date is maintained on the TOE. +3.3 Identification and Authentication +The IDM console application provides user interfaces that administrators may use to +manage TOE functions. The operating system and the database in the TOE Environ- +ment are queried to individually authenticate administrators or users. The TOE main- +tains authorization information that determines which TOE functions an authenticated +administrators or users (of a given role) may perform. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +7 (18) +The TOE maintains the following list of security attributes belonging to individual us- +ers: + User Identity (i.e., user name) + Authentication Status (whether the IT Environment validated the username/pass- +word) + Privilege Level (Administrator or User) +3.4 User Data Protection +The TOE implements a discretionary access control policy to define what roles can +access particular functions of the TOE. All access and actions for system reports, com- +ponent audit logs, TOE configuration, operator account attributes (defined in +FIA_ATD.1) are protected via access control list. When a user requests to perform an +action on an object, the TOE verifies the role associated with the user name. Access is +granted if the user (or group of users) has the specific rights required for the type of +operation requested on the object. +Identity Manager can enforce password policies on incoming passwords from con- +nected systems and on passwords set or changed through the User Application pass- +word self-service. If the new password does not comply, you can specify that Identity +Manager not accept the password. This also means that passwords that don't comply +with your policies are not distributed to other connected systems. +In addition, can enforce password policies on connected systems. If the password be- +ing published to the Identity Vault does not comply with rules in a policy, you can +specify that Identity Manager not only does not accept the password for distribution, +but actually resets the noncompliant password on the connected system by using the +current Distribution password in the Identity Vault. +3.5 Trusted Path / Channel +The TOE provides a trusted channel between the TOE and external web servers. +The TOE provides a trusted path for TOE administrators and TOE users to communi- +cate with the TOE. The trusted path is implemented using HTTPS. The TOE's imple- +mentation of TLS is described in the previous section (Trusted Channel). +3.6 Cryptographic Support +Cryptographic protection of data in transit between the TOE and remote users, and be- +tween the TOE and external web servers is provided by the OpenSSL FIPS Object +Module software version 2.0.10 (Cryptographic Module Validation Program (CMVP) +certificate number 1747) libraries. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +8 (18) +4 Assumptions and Clarification of Scope +4.1 Usage Assumptions +The Security Target [ST] makes two assumptions on the usage of the TOE. +A.MANAGE - Administrators of the TOE are assumed to be appropriately trained to +undertake the installation, configuration and management of the TOE in a secure and +trusted manner. +A.NOEVIL - Administrators of the TOE and users on the local area network are not +careless, willfully negligent, nor hostile, and will follow and abide by the instructions +provided by the TOE documentation +4.2 Environmental Assumptions +The Security Target [ST] makes three assumptions on the operational environment of +the TOE. +A.LOCATE - The processing platforms on which the TOE resides are assumed to be +located within a facility that provides controlled access +A.CONFIG - The TOE is configured to receive all passwords and associated data from +network-attached systems. +A.TIMESOURCE - The TOE has a trusted source for system time via NTP server +4.3 Clarification of Scope +The Security Target contains five threats, which have been considered during the eval- +uation. +T.NO_AUTH - An unauthorized user may gain access to the TOE and alter the TOE +configuration. +T.NO_PRIV - An authorized user of the TOE exceeds his/her assigned security privi- +leges resulting in unauthorized modification of the TOE configuration and/or data. +T.USER_ACCESS_DENY - An authorized user may be able to change user authenti- +cation data and or user access policies and deny their access to it later. +T.PASSWD_COMPROMISE - An unauthorized user may be able to obtain and use +user passwords. +T.PROT_TRANS - An unauthorized user may be able to gather information from +communications between components. +The Security Target contains one Organisational Security Policies (OSPs), which have +been considered during the evaluation. +P.REMOTE_DATA - Passwords and account information from network-attached sys- +tems shall be monitored and managed. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +9 (18) +5 Architectural Information +The TOE consists of the following components: + Administration Workstation (Console)2 + Identity Applications (RBPM) + Designer aka Identity Manager Designer + Analyzer aka Identity Manager Analyzer + Identity Manager + Identity Manager Engine + Identity Vault + iManager + Reporting Server + Identity Reporting Module + Log Manager + Sentinel Log Management for Identity Governance and Administration + SSO Provider + One SSO Provider (OSP) + Self Service Password Reset + Self Service Password Reset (SSPR) +Figure 1, TOE Deployment with subsystems +The TOE provides the following functions: data synchronization, role management, +auditing/reporting, and management. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +10 (18) + Data synchronization, including password synchronization, is provided by the +base components of the Identity Manager solution: the Identity Vault, Identity +Manager engine, drivers, Remote Loader, and connected applications + Role management is provided by the User Application + Auditing and reporting are provided by the Identity Reporting Module + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +11 (18) +6 Documentation +The TOE includes the following guidance documentation: + Quick Start Guide for Installing NetIQ Identity Manager 4.7 February 2018 +[QSIM] + NetIQ Identity Manager Setup Guide for Linux February 2018 [SUL] + NetIQ Identity Manager 4.7, Operational User Guidance and Preparative Proce- +dures Supplement (AGD-IGS), version 0.6, is supplied for those customers that +need guidance on how to set the TOE in the evaluated configuration. [AGD] + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +12 (18) +7 IT Product Testing +7.1 Developer Testing +There are 30 test cases covering all SFRs with at least one test per SFR. All tests were +successful with a pass verdict. +7.2 Evaluator Testing +Since all SFRs and security function requirements were tested by the developer the +evaluator focused on repetition of the developer's test cases and penetration testing. +7.3 Penetration Testing +Port and vulnerability scan were performed on Identity manager engine, Identity appli- +cations (RBPM), and Identity reporting module. +No unforeseen ports or vulnerabilities were found. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +13 (18) +8 Evaluated Configuration +The TOE consists of a set of software applications run on one or multiple distributed +systems. The TOE requires the following software components as part of the evalu- +ated configuration: +Component Requirements +Administration Workstation Mozilla Firefox 65 +Identity Applications (RBPM) +Designer / Analyzer) +SUSE Linux Enterprise Server 12 SP4 +Identity Manager (Identity Man- +ager Engine) +SUSE Linux Enterprise Server 12 SP4 +Reporting Server (Identity Re- +porting Module) +SUSE Linux Enterprise Server 12 SP4 +Log Manager (Sentinel Log Man- +agement for Identity Governance +and Administration) +SUSE Linux Enterprise Server 12 SP4 +SSO Provider (OneSSO Provider) SUSE Linux Enterprise Server 12 SP4 +Self Service Password Reset SUSE Linux Enterprise Server 12 SP4 +In addition to the platform requirements mentioned above, the following hardware re- +sources are needed in order to install and configure Identity Manager on each plat- +form: + A minimum of 8 GB RAM + 15 GB available disk space to install all the components. + Additional disk space to configure and populate data. This might vary depending +on your connected systems and number of objects in the Identity Vault. +For server-based components, it is recommended that the platform have a minimum of +2 CPUs or cores. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +14 (18) +9 Results of the Evaluation +The evaluators applied each work unit of the Common Methodology [CEM] within +the scope of the evaluation, and concluded that the TOE meets the security objectives +stated in the Security Target [ST] for an attack potential of Basic. +The certifier reviewed the work of the evaluators and determined that the evaluation +was conducted in accordance with the Common Criteria [CC]. +The evaluators' overall verdict is PASS. +The verdicts for the respective assurance classes and components are summarised in +the following table: +Assurance Class/Family Short name Verdict +Development ADV: PASS +Security architecture description ADV_ARC.1 PASS +Functional specification with complete summary ADV_FSP.3 PASS +Architectural design ADV_TDS.2 PASS +Guidance documents AGD: PASS +Operational user guidance AGD_OPE.1 PASS +Preparative procedures AGD_PRE.1 PASS +Life-cycle support ALC: PASS +Authorisation controls ALC_CMC.3 PASS +Implementation representation CM coverage ALC_CMS.3 PASS +Delivery procedures ALC_DEL.1 PASS +Identification of security measures ALC_DVS.1 PASS +Developer defined life-cycle model ALC_LCD.1 PASS +Flaw reporting procedures ALC_FLR.2 PASS +Security Target evaluation ASE: PASS +Conformance claims ASE_CCL.1 PASS +Extended components definition ASE_ECD.1 PASS +ST introduction ASE_INT.1 PASS +Security objectives ASE_OBJ.2 PASS +Derived security requirements ASE_REQ.2 PASS +Security problem definition ASE_SPD.1 PASS +TOE summary specification ASE_TSS.1 PASS +Tests ATE: PASS +Analysis of coverage ATE_COV.2 PASS +Testing: basic design ATE_DPT.1 PASS +Functional testing ATE_FUN.1 PASS +Independent testing - sample ATE_IND.2 PASS +Vulnerability assessment AVA: PASS +Vulnerability analysis AVA_VAN.2 PASS + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +15 (18) +10 Evaluator Comments and Recommendations +None. + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +16 (18) +11 Glossary +CC Common Criteria version 3.1 +EAL Evaluation Assurance Level +FIPS Federal Information Processing Standard +IDM Identity Manager +ITSEF +IT Security Evaluation Facility, test labora- +tory licensed to operate within a evaluation +and certification scheme +NTP Network Time Protocol +OSP Organizational Security Policy +OSP One SSO Provider +SSO Single Sign On +SFP Security Function Policy +SFR Security Functional Requirement +SSPR Self Service Password Reset +ST Security Target +TOE Target of Evaluation + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +17 (18) +12 Bibliography +ST NetIQ Identity Manager 4.7 Security Target (ST), NetIQ +Corporation, 2020-06-01, document version 2.6 +QSIM Quick Start Guide for Installing NetIQ Identity Manager 4.7 +February 2018 +SUL NetIQ Identity Manager Setup Guide for Linux February 2018 +AGD NetIQ Identity Manager 4.7, Operational User Guidance and +Preparative Procedures Supplement (AGD-IGS), version 0.6 +CCpart1 Common Criteria for Information Technology Security Evaluation, +Part 1, version 3.1 revision 5, CCMB-2017-04-001 +CCpart2 Common Criteria for Information Technology Security Evaluation, +Part 2, version 3.1 revision 5, CCMB-2017-04-002 +CCpart3 Common Criteria for Information Technology Security +Evaluation,Part 3, version 3.1 revision 5, CCMB-2017-04-003 +CC CCpart1 + CCpart2 + CCpart3 +CEM Common Methodology for Information Technology Security +Evaluation, version 3.1 revision 5, CCMB-2017-04-004 +SP-002 SP-002 Evaluation and Certification, CSEC, 2019-09-24, document +version 31.0 + Swedish Certification Body for IT Security +Certification Report NetIQ® Identity Manager 4.7 +18FMV7705-43:1 1.0 2020-06-15 +18 (18) +Appendix A Scheme Versions +During the certification the following versions of the Swedish Common Criteria Eval- +uation and Certification scheme have been used. +A.1 Scheme/Quality Management System +During the certification project, the following versions of the quality management sys- +tem (QMS) have been applicable since the certification application was received: +QMS 1.21.5 valid from 2018-11-19 +QMS 1.22 valid from 2019-02-01 +QMS 1.22.1 valid from 2019-03-08 +QMS 1.22.2 valid from 2019-05-02 +QMS 1.22.3 valid from 2019-05-20 +QMS 1.23 valid from 2019-10-14 +QMS 1.23.1 valid from 2020-03-06 +QMS 1.23.2 valid from 2020-05-11 +In order to ensure consistency in the outcome of the certification, the certifier has ex- +amined the changes introduced in each update of the quality management system. +The changes between consecutive versions are outlined in "Ändringslista CSEC QMS +1.23.1". The certifier concluded that, from QMS 1.21.5 to the current QMS 1.23.2, +there are no changes with impact on the result of the certification. +Note that the SP-188 Scheme Crypto Policy version 9.0 was introduced in QMS 1.23. +The certification application was submitted before the SP-188 Scheme Crypto Policy +version 9.0 was introduced and therefore version 8.0 was used. +A.2 Scheme Notes +The following Scheme interpretations have been considered during the certification. + Scheme Note 15 - Demonstration of test Coverage + Scheme Note 18 - Highlighted Requirements on the Security Target + Scheme Note 22 - Vulnerability assessment + Scheme Note 28 - Updated procedures for application, evaluation and certification + \ No newline at end of file diff --git a/test/data/test_cc_oop/target_869415cc4b91282e.txt b/test/data/test_cc_oop/target_869415cc4b91282e.txt new file mode 100644 index 00000000..9435c203 --- /dev/null +++ b/test/data/test_cc_oop/target_869415cc4b91282e.txt @@ -0,0 +1,1497 @@ +NetIQ Identity Manager 4.7 +Security Target (ST) +Date: June 1, 2020 +Version: 2.6 +Prepared By: NetIQ Corporation +Prepared For: NetIQ Corporation +515 Post Oak Blvd +Suite 1200 +Houston, Texas 77027 +Abstract +This document provides the basis for an evaluation of a specific Target of Evaluation (TOE), Identity +Manager 4.7. This Security Target (ST) defines a set of assumptions about the aspects of the environment, +a list of threats that the product intends to counter, a set of security objectives, a set of security requirements +and the IT security functions provided by the TOE which meet the set of requirements. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 2 of 36 +Table of Contents +Table of Contents...................................................................................................................................2 +List of Tables.........................................................................................................................................3 +List of Figures........................................................................................................................................4 +1. Introduction ...........................................................................................................................................5 +Security Target Reference:............................................................................................................5 +TOE Reference..............................................................................................................................5 +Document Organization................................................................................................................5 +Document Conventions.................................................................................................................6 +Document Terminology................................................................................................................6 +TOE Overview..............................................................................................................................7 +TOE Description...........................................................................................................................8 +Administration Workstation (Console):........................................................................................8 +Identity Applications (RBPM)......................................................................................................8 +Identity Manager:..........................................................................................................................9 +Reporting Server:..........................................................................................................................9 +Log Manager:................................................................................................................................9 +OneSSO Provider:.........................................................................................................................9 +Self Service Password Reset:......................................................................................................10 +TOE Delivery:.............................................................................................................................10 +TOE Environment.......................................................................................................................10 +Virtual Machines.........................................................................................................................10 +Hardware and Software Supplied by the IT Environment..........................................................11 +Logical Boundary........................................................................................................................11 +TOE Security Functional Policies...............................................................................................12 +Discretionary Access Control SFP..............................................................................................12 +TOE Vendor Documentation / Guidance....................................................................................12 +Features / Functionality NOT Included in the TOE....................................................................12 +2. Conformance Claims ...........................................................................................................................14 +CC Conformance Claim..............................................................................................................14 +PP Claim .....................................................................................................................................14 +Package Claim ............................................................................................................................14 +Conformance Rationale...............................................................................................................14 +3. Security Problem Definition................................................................................................................15 +Threats.........................................................................................................................................15 +Organizational Security Policies.................................................................................................15 +Assumptions................................................................................................................................15 +4. Security Objectives..............................................................................................................................17 +Security Objectives for the TOE.................................................................................................17 +Security Objectives for the Operational Environment................................................................17 +Security Objectives Rationale.....................................................................................................17 +Mapping of Objectives................................................................................................................18 +5. Extended Components Definition........................................................................................................20 +6. Security Requirements.........................................................................................................................21 +Security Functional Requirements..............................................................................................21 +Security Audit (FAU) .................................................................................................................21 +FAU_GEN.1 Audit Data Generation..........................................................................................21 +FAU_SAR.1 Audit Review.........................................................................................................22 + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 3 of 36 +Cryptographic Support................................................................................................................22 +FCS_CKM.1 Cryptographic key generation...............................................................................22 +FCS_CKM.4 Cryptographic key destruction..............................................................................22 +FCS_COP.1 Cryptographic operation (Encryption / Decryption) ..............................................22 +Information Flow Control (FDP) ................................................................................................23 +FDP_ACC.1 Subset Access Control...........................................................................................23 +FDP_ACF.1 Security Attribute Based Access Control...............................................................23 +Identification and Authentication (FIA) .....................................................................................24 +FIA_ATD.1 ­ User Attribute Definition.....................................................................................24 +FIA_UAU.2 User Authentication before Any Action ................................................................24 +FIA_UID.2 User Identification before Any Action....................................................................24 +Security Management (FMT)......................................................................................................24 +FMT_MSA.1 Management of security attributes .......................................................................24 +FMT_MSA.2 Secure Security Attributes....................................................................................24 +FMT_MSA.3 Static Attribute Initialization................................................................................24 +FMT_MTD.1 Management of TSF Data....................................................................................25 +FMT_SMF.1 Specification of Management Functions ..............................................................25 +FMT_SMR.1 Security Roles.......................................................................................................25 +Protection of the TSF (FPT)........................................................................................................25 +FPT_TDC.1 Inter-TSF Basic TSF Data Consistency .................................................................25 +Trusted Path / Channel (FTP) .....................................................................................................26 +FTP_ITC.1 Inter-TSF trusted channel ........................................................................................26 +FTP_TRP.1 Trusted Path............................................................................................................26 +Security Assurance Requirements ..............................................................................................26 +Security Requirements Rationale................................................................................................26 +Security Functional Requirements..............................................................................................26 +Dependency Rationale ................................................................................................................27 +Sufficiency of Security Requirements ........................................................................................28 +Security Assurance Requirements ..............................................................................................30 +Security Assurance Requirements Rationale ..............................................................................30 +Security Assurance Requirements Evidence...............................................................................31 +7. TOE Summary Specification...............................................................................................................33 +TOE Security Functions..............................................................................................................33 +Security Audit.............................................................................................................................33 +Identification and Authentication................................................................................................33 +User Data Protection...................................................................................................................33 +Security Management .................................................................................................................34 +Trusted Path / Channels ..............................................................................................................35 +Trusted Channel..........................................................................................................................35 +Trusted Path:...............................................................................................................................35 +Cryptographic Support................................................................................................................35 +List of Tables +Table 1 ­ ST Organization and Section Descriptions...................................................................................6 +Table 2 ­ Acronyms Used in Security Target...............................................................................................7 +Table 3 ­ CAVP Certificate Numbers ..........................................................................................................9 +Table 4 ­ Virtual Machine Environment Requirements .............................................................................11 +Table 5 ­ IT Environment Component Requirements................................................................................11 +Table 6 ­ Logical Boundary Descriptions ..................................................................................................12 +Table 7 ­ IT Environment Components - Not In TOE ...............................................................................13 + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 4 of 36 +Table 8 ­ Threats Addressed by the TOE...................................................................................................15 +Table 9 ­ Organizational Security Policies.................................................................................................15 +Table 10 ­ Assumptions..............................................................................................................................16 +Table 11 ­ TOE Security Objectives ..........................................................................................................17 +Table 12 ­ Operational Environment Security Objectives .........................................................................17 +Table 13 ­ Mapping of Assumptions, Threats, Policies and ORSP s to Security Objectives.....................18 +Table 14 ­ Mapping of Threats, Policies, and Assumptions to Objectives ................................................19 +Table 15 ­ TOE Security Functional Requirements ...................................................................................21 +Table 16 ­ Cryptographic Standards...........................................................................................................22 +Table 17 ­ Cryptographic Operations.........................................................................................................23 +Table 18 ­ Management of TSF data..........................................................................................................25 +Table 19 ­ Mapping of TOE Security Functional Requirements and Objectives.......................................27 +Table 20 ­ Mapping of SFR to Dependencies and Rationales....................................................................28 +Table 20 ­ Rationale for TOE SFRs to Objectives.....................................................................................30 +Table 22 ­ Security Assurance Requirements at EAL3..............................................................................30 +Table 23 ­ Security Assurance Rationale and Measures ............................................................................32 +Table 24 ­ Roles and Functions..................................................................................................................34 +Table 22 ­ CAVP........................................................................................................................................36 +List of Figures +Figure 1 ­ TOE Deployment with Subsystems.............................................................................................7 +Figure 2 ­ Sample Download List ..............................................................................................................10 + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 5 of 36 +1. Introduction +This section identifies the Security Target (ST), Target of Evaluation (TOE), Security Target +organization, document conventions, and terminology. It also includes an overview of the +evaluated product. +Security Target Reference: +ST Title NetIQ Identity Manager 4.7 Security Target: +ST Revision 2.6 +ST Publication Date June 1, 2020 +ST Author Michael F. Angelo +TOE Reference +TOE Reference NetIQ Identity Manager 4.7 +TOE Developer NetIQ Corporation +Evaluation Assurance Level (EAL) EAL3+ +Note: The file download name is: Identity_Manager_4.7_Linux.iso . +Note: The official name of the product is NetIQ Identity Manager 4.7 Advanced Edition. The +released product can be uniquely identified as: NetIQ Identity Manager 4.7.3. The product name +may also be abbreviated as Identity Manager 4.7 AE, Identity Manager, IDM 4.7.3AE or IDM 4.7 +or simply IDM . Finally the TOE, if examined for the build number will be identified as NetIQ +Identity Manager 4.7.3.0.317. For the purpose of this document all of the above references are +equivalent, and the document may refer to the product simply as IDM or the TOE. +Document Organization +This Security Target follows the following format: +SECTION TITLE DESCRIPTION +1 Introduction Provides an overview of the TOE and defines the +hardware and software that make up the TOE as well +as the physical and logical boundaries of the TOE +2 Conformance Claims Lists evaluation conformance to Common Criteria +versions, Protection Profiles, or Packages where +applicable +3 Security Problem +Definition +Specifies the threats, assumptions and organizational +security policies that affect the TOE +4 Security Objectives Defines the security objectives for the +TOE/operational environment and provides a +rationale to demonstrate that the security objectives +satisfy the threats +5 Extended +Components +Definition +Describes extended components of the evaluation (if +any) +6 Security +Requirements +Contains the functional and assurance requirements +for this TOE + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 6 of 36 +SECTION TITLE DESCRIPTION +7 TOE Summary +Specification +Identifies the IT security functions provided by the +TOE and also identifies the assurance measures +targeted to meet the assurance requirements. +Table 1 ­ ST Organization and Section Descriptions +Document Conventions +The notation, formatting, and conventions used in this Security Target are consistent with those +used in Version 3.1 of the Common Criteria. Selected presentation choices are discussed here +to aid the Security Target reader. The Common Criteria allows several operations to be +performed on functional requirements: The allowable operations defined in Part 2 of the +Common Criteria are refinement, selection, assignment and iteration. + The refinement operation is used to add detail to a requirement, and thus further +restricts a requirement. Refinement of security requirements is denoted by bold text. +Any text removed is indicated with a strikethrough format (Example: TSF). + The selection operation is picking one or more items from a list in order to narrow the +scope of a component element. Selections are denoted by italicized text. + The assignment operation is used to assign a specific value to an unspecified parameter, +such as the length of a password. An assignment operation is indicated by showing the +value in square brackets, i.e. [assignment_value(s)]. + Iterated functional and assurance requirements are given unique identifiers by +appending to the base requirement identifier from the Common Criteria an iteration +number inside parenthesis, for example, FMT_MTD.1.1 (1) and FMT_MTD.1.1 (2) refer +to separate instances of the FMT_MTD.1 security functional requirement component. +When not embedded in a Security Functional Requirement, italicized text is used for both +official document titles and text meant to be emphasized more than plain text. +Document Terminology +The following table describes the acronyms used in this document: +TERM DEFINITION +CC Common Criteria version 3.1 +EAL Evaluation Assurance Level +IDM Identity Manager +IDV Identity Vault +IGA Identity Governance and Administration +NMAS NetIQ Modular Authentication Service +NTP Network Time Protocol +ORSP Organizational Security Policy +OSP One SSO Provider +SSO Single Sign On +SFP Security Function Policy +SFR Security Functional Requirement +SLM Sentinel Log Manager + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 7 of 36 +TERM DEFINITION +SSPR Self Service Password Reset +ST Security Target +TOE Target of Evaluation +TSF TOE Security Function +Table 2 ­ Acronyms Used in Security Target +TOE Overview +The TOE is NetIQ Identity Manager 4.7. NetIQ Identity Manager provides data sharing and +synchronization services which enable applications, directories, and databases to share +information. It links scattered information and enables you to establish policies that govern +automatic updates to designated systems when identity changes occur. +Identity Manager provides the foundation for account provisioning, security, single sign-on, +user self-service, authentication, authorization, automated workflow, and Web services. It +allows you to integrate, manage, and control your distributed identity information so you can +securely deliver the right resources to the right people. +The following diagram shows a typical TOE deployment: +Identity Reporting Module +Operating System +General Purpose Computing +Platform +Reporting Server +Sentinel Log Management +for Identity Governance +and Administration +Operating System +General Purpose Computing +Platform +Log Manager +Identity Manager Engine +Identity Vault +Operating System +General Purpose Computing +Platform +Identity Applications +(RBPM) +Web Browser +Operating System +General Purpose Computing +(GPC) Platform +Identity Application +4 +1 +2 +8 +10 += TOE Component += IT Environment Component +One SSO Provider +(uname / pass, Kerberos, +SAML) +Operating System +General Purpose Computing +Platform +3 +5 +SSO Provider +Self Service Password +Reset +Web Browser +Operating System +General Purpose Computing +Platform +Self Service Password Reset +11 +9 +12 +7a +Identity Manager +6 +B +Administration +Workstation +(Console) 7b +Separate communication paths to Sentinel Log Manager +7a ­ Identity Vault to Sentinel Log Manager +7b ­ iManager to Sentinel Log Manager +C +A +iManager +Designer / Analyzer += TOE Sub Component +OpenSSL +Figure 1 ­ TOE Deployment with Subsystems1 +The TOE provides the following functions: data synchronization, role management, +auditing/reporting, and management. +11 +Note the Administration Workstation Console is not included in the evaluation as there is no code that is added to it to make it +explicitly a workstation console. It is included in the document as a component required for access.to the TOE. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 8 of 36 + Data synchronization, including password synchronization, is provided by the base +components of the Identity Manager solution: the Identity Vault, Identity Manager +engine, drivers, Remote Loader, and connected applications + Role management is provided by the User Application + Auditing and reporting are provided by the Identity Reporting Module +TOE Description +NetIQ Identity Manager 4.7 is a comprehensive identity management suite. It provides an +intelligent identity framework that leverages your existing IT assets and new computing +models like Software as a Service (SaaS) by reducing cost and ensuring compliance across +physical, virtual, and cloud environments. With the NetIQ Identity Manager solution, you can +make sure that your business has the most current user identity information. You can retain +control at the enterprise level by managing, provisioning, and de-provisioning identities within +the firewall and extending to the cloud. Through streamlined user administration and +processes, Identity Manager helps organizations reduce management costs, increase +productivity and security, and comply with government regulations. +The TOE is a software TOE and includes the following functions. +Each function contains the components as follows: +1. Administration Workstation (Console)2 +2. Identity Applications (RBPM) 4.7.3.0.1109 + Designer aka Identity Manager Designer 4.7.3.0.20190614 + Analyzer aka Identity Manager Analyzer +3. Identity Manager + Identity Manager Engine 4.7.3.0.AE +o Identity Vault 9.1.4 +o iManager 3.1.4 +4. Reporting Server + Identity Reporting Module 6.5.0. F14508F +5. Log Manager + Sentinel Log Management for Identity Governance and Administration 8.2.2.0_5415 +6. SSO Provider + One SSO Provider (OSP) 6.3.3.0 +7. Self Service Password Reset + Self Service Password Reset (SSPR) 4.4.0.2 B366 r39762 +Administration Workstation (Console): +The Administration Workstation (Console) is used to access the Identity Applications (RBPM), +Identity Manager, and the Reporting Server. Each of these functions is described below. +Identity Applications (RBPM) +The Identity Applications (RBPM) houses the Designer / Analyzer functions. The Identity +Application is a Web application (browser-based) that gives users and business administrators +the ability to perform a variety of identity self-service and roles provisioning tasks, including +managing passwords and identity data, initiating and monitoring provisioning and role +assignment requests, managing the approval process for provisioning requests, and verifying +2 +The Administration Workstation (Console) is not part of the TOE, in that there is no code added to it in order to function as the +Console it is required to access features and function of the TOE and is included for completeness. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 9 of 36 +attestation reports. It includes the workflow engine that controls the routing of requests +through the appropriate approval process. Designer aka Designer for Identity Manager helps +you design, test, document, and deploy Identity Manager solutions in a network or test +environment. Analyzer aka NetIQ Analyzer for Identity Manager is an identity management +toolset that helps you ensure that internal data quality policies are adhered to by providing +data analysis, data cleansing, data reconciliation, and data monitoring/reporting. Analyzer lets +you analyze, enhance, and control all data stores throughout the enterprise. +Identity Manager: +The Identity Manager houses the Identity Manager Engine (and the Identity Vault which +contains the Identity Applications data) and iManager. The Identity Manager Engine +synchronizes identity data between applications. For example, data synchronized from a +PeopleSoft system to Lotus Notes is first added to the Identity Vault and then sent to the Lotus +Notes system. In addition, the Identity Vault stores information specific to Identity Manager, +such as driver configurations, parameters, and policies. +The following packages are used to provide cryptographic functions, and are not included in +the TOE boundary. NetIQ eDirectory is used for the Identity Vault. eDirectory provides access +to the OpenSSL Cryptographic functionality. +They meet the cryptographic quality requirements as evidenced by the following certificates: +Component CAVP Cert # +AES Certs. #3090 and #3264 +HMAC Certs. #1937 and #2063 +RSA Certs. #1581 and #1664 +Table 3 ­ CAVP Certificate Numbers +Reporting Server: +The reporting server houses the Identity Reporting Module. The Identity Reporting Module +generates reports that show critical business information about various aspects of your +Identity Manager configuration, including information collected from Identity Vaults and +managed systems such as Active Directory or SAP. The reporting module provides a set of +predefined report definitions you can use to generate reports. In addition, it gives you the +option to import custom reports defined in a third-party tool. The user interface for the +reporting module makes it easy to schedule reports to run at off-peak times to optimize +performance. +The IDM Tools are used to manage the Identity Manager solution. This includes functions to: + Analyze, enhance, and control all data stores throughout the enterprise + Design, deploy, and document the TOE + Manage Identity Manager and receive real-time health and status information +about the Identity Manager system + Define and maintain which authorizations are associated with which business roles +Log Manager: +The Log Manager, also known as Sentinel Log Manager for Identity Governance and +Administration (SLM for IGA), collects and acknowledges receipt of auditing data from all +aspects of the product. +OneSSO Provider: + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 10 of 36 +The OneSSO Provider, also known as OSP) is a single interface for access authentication. This +provider can handle user name / password, Kerberos, and SAML tokens. +Self Service Password Reset: +Self Service Password Reset (SSPR) allows users to enroll, update, and reset their passwords +without administrative intervention in the Identity Vault (IDV). +Note: that the components above can be installed on one or multiple distributed systems. Also, +the hardware, operating systems and third-party support software (e.g. DBMS) on each of the +systems are excluded from the TOE boundary. +TOE Delivery: +The TOE software is provided to customers via secure download from the download portal +(https://dl.netiq.com/index.jsp). The software is available as either a gnu zip (.gz), iso +formatted optical disk (.iso). zip (.zip) or dmg (if mac) depending on your destination platform. +Once downloaded, and extracted, the setup files can be executed to perform the installation. +Figure 2 ­ Sample Download List +TOE Environment +Virtual Machines +The following TOE components can be installed in virtual machines (VM). + Console / Administration Workstation (Identity Applications) + Identity Manager + Reporting Server + Sentinel Log Manager + One SSO Provider + Self Service Password Reset (SSPR) +The hardware and software requirements for the operational environment to support the VM +are listed in the table below: + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 11 of 36 +Category Console / +Administration +Workstation +(Identity +Applications3) +Identity +Manager +(Identity +Manager +Engine) +Reporting +Server +(Identity +Reporting +Module) +Log +Manager +(SLM for +Identity +Gov & +Adm) +SSO +Provider +(OneSSO +Provider) +Self Service +Password Reset +(SSPR) +Processor 2 CPU cores 2 CPU +cores +2 CPU +cores +4 to 8 CPU +cores +2 CPU +cores +2 CPU cores +Memory 8 GB 8 GB 8 GB 8 to 16 GB 8 GB 8 GB +Table 4 ­ Virtual Machine Environment Requirements +Hardware and Software Supplied by the IT Environment +The TOE consists of a set of software applications run on one or multiple distributed systems. +The TOE requires the following software components as part of the evaluated configuration: +Component Requirements +Administration Workstation Mozilla Firefox 65 +Identity Applications (RBPM) +Designer / Analyzer) +SUSE Linux Enterprise Server 12 SP4 +Identity Manager (Identity +Manager Engine) +SUSE Linux Enterprise Server 12 SP4 +Reporting Server (Identity +Reporting Module) +SUSE Linux Enterprise Server 12 SP4 +Log Manager (Sentinel Log +Management for Identity +Governance and +Administration) +SUSE Linux Enterprise Server 12 SP4 +SSO Provider (OneSSO +Provider) +SUSE Linux Enterprise Server 12 SP4 +Self Service Password Reset SUSE Linux Enterprise Server 12 SP4 +Table 5 ­ IT Environment Component Requirements +In addition to the platform requirements mentioned above, the following hardware resources +are needed in order to install and configure Identity Manager on each platform: + A minimum of 8 GB RAM + 15 GB available disk space to install all the components. + Additional disk space to configure and populate data. This might vary depending +on your connected systems and number of objects in the Identity Vault. +For server-based components, it is recommended that the platform have a minimum of 2 CPUs +or cores +Logical Boundary +This section outlines the boundaries of the security functionality of the TOE; the logical +boundary of the TOE includes the security functionality described in the following table: +3 +The system requirements also apply to the following components that you use with the identity applications: PostgreSQL, Tomcat, +NetIQ One SSO Provider (OSP), and NetIQ Self Service Password Reset. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 12 of 36 +TSF DESCRIPTION +Security +Management +The TOE restricts the ability to enable, modify and disable security +policy rules and user roles to an authorized Administrator. The TOE +also provides the functions necessary for effective management of +the TOE security functions. Administrators configure the TOE with +the Management Console via Web-based connection. +Security Audit The TOE supports the provision of log data from each system +component, such as user login/logout and incident/ticket +management actions. It also records security events such as failed +login attempts, etc. Audit trails can be stored for later review and +analysis. +Cryptographic +Support +The TOE utilizes the OpenSSL cryptographic module to provide +support for HTTPS / TLS communications with administrators and +TOE components. +Identification and +Authentication +The TOE enforces individual I&A. Operators must successfully +authenticate using a unique identifier and password prior to +performing any actions on the TOE. +User Data +Protection +The TOE enforces discretionary access rules using an access control +list with user attributes. +Trusted Path / +Channels +The TOE utilizes HTTPS/TLS to provide trusted paths and inter-TSF +trusted channels. +Table 6 ­ Logical Boundary Descriptions +TOE Security Functional Policies +The TOE supports the following Security Functional Policy: +Discretionary Access Control SFP +The TOE implements an access control SFP named Discretionary Access Control SFP. This SFP +determines and enforces the privileges associated with operator roles. An authorized +administrator can define specific services available to administrators and users via the +Management Console. +TOE Vendor Documentation / Guidance +In addition to the documentation generated for the certification, the TOE includes the following +product and guidance documentation generated by NetIQ: + Quick Start Guide for Installing NetIQ Identity Manager 4.7 February 2018 + NetIQ Identity Manager Setup Guide for Linux February 2018 + NetIQ Identity Manager 4.7, Operational User Guidance and Preparative Procedures +Supplement (AGD-IGS), version 0.6, is supplied for those customers that need +guidance on how to set the TOE in the evaluated configuration. +Features / Functionality NOT Included in the TOE +The following supported operating systems and software were not included in the evaluated +configuration: +Functions Requirements +Administration Workstation (Console) Web Browsers + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 13 of 36 +Functions Requirements + Internet Explorer 11 + Google Chrome +Identity Applications (Includes Designer / +Analyzer) +RHEL 7.5 +Windows Server 2016 +Identity Manager (Includes Identity Vault +and, iManager) +RHEL 7.5 +Windows Server 2016 +Reporting Server +(includes Identity Reporting Module) +RHEL 7.5 +Windows Server 2016 +Log Manager (includes Sentinel Log +Management for Identity Governance and +Administration) +RHEL 7.5 +One SSO Provider (uname / pass, Kerberos, +SAML) +RHEL 7.5 +Windows Server 2016 +Self Service Password Reset (SSPR) RHEL 7.5 +Windows Server 2016 +Table 7 ­ IT Environment Components - Not In TOE + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 14 of 36 +2. Conformance Claims +CC Conformance Claim +The TOE is Common Criteria Version 3.1 Revision 5 (April 2017) Part 2 conformant and Part 3 +conformant. +PP Claim +The TOE does not claim conformance to any registered Protection Profile. +Package Claim +The TOE claims conformance to the EAL3 assurance package defined in Part 3 of the Common +Criteria Version 3.1 Revision 5 (April 2017). The TOE does not claim conformance to any +functional package. The TOE EAL3 assurance package is augmented with ALC_FLR.2 +Conformance Rationale +No conformance rationale is necessary for this evaluation since this Security Target does not +claim conformance to a Protection Profile. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 15 of 36 +3. Security Problem Definition +In order to clarify the nature of the security problem that the TOE is intended to solve, this +section describes the following: + Any known or assumed threats to the assets against which specific protection within the TOE or +its environment is required + Any organizational security policy statements or rules with which the TOE must comply + Any assumptions about the security aspects of the environment and/or of the manner in which +the TOE is intended to be used. +This chapter identifies assumptions as A.assumption, threats as T.threat and policies as P.policy. +Threats +The following are threats identified for the TOE and the IT System (or operating environment) +the TOE monitors. The TOE itself has threats and the TOE is also responsible for addressing +threats to the environment in which it resides. The assumed level of expertise of the attacker +for all threats is unsophisticated. +The TOE addresses the following threats: +THREAT DESCRIPTION +T.NO_AUTH An unauthorized user may gain access to the TOE and alter the +TOE configuration. +T.NO_PRIV An authorized user of the TOE exceeds his/her assigned +security privileges resulting in unauthorized modification of the +TOE configuration and/or data. +T.USER_ACCESS_DENY An authorized user may be able to change user authentication data +and or user access policies and deny their access to it later. +T.PASSWD_COMPROMISE An unauthorized user may be able to obtain and use user +passwords. +T.PROT_TRANS An unauthorized user may be able to gather information from +communications between components. +Table 8 ­ Threats Addressed by the TOE +Organizational Security Policies +The TOE meets the following organizational security policies: +ASSUMPTION DESCRIPTION +P.REMOTE_DATA Passwords and account information from network-attached systems +shall be monitored and managed. +Table 9 ­ Organizational Security Policies +Assumptions +The TOE is assured to provide effective security measures in a co-operative non-hostile +environment only if it is installed, managed, and used correctly. The following specific +conditions are assumed to exist in an environment where the TOE is employed. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 16 of 36 +ASSUMPTION DESCRIPTION +A.MANAGE Administrators of the TOE are assumed to be appropriately trained to +undertake the installation, configuration and management of the TOE +in a secure and trusted manner. +A.NOEVIL Administrators of the TOE and users on the local area network are not +careless, willfully negligent, nor hostile, and will follow and abide by the +instructions provided by the TOE documentation +A.LOCATE The processing platforms on which the TOE resides are assumed to be +located within a facility that provides controlled access +A.CONFIG The TOE is configured to receive all passwords and associated data +from network-attached systems. +A.TIMESOURCE The TOE has a trusted source for system time via NTP server +Table 10 ­ Assumptions + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 17 of 36 +4. Security Objectives +Security Objectives for the TOE +The IT security objectives for the TOE are addressed below: +OBJECTIVE DESCRIPTION +O.MANAGE_DATA The TOE shall provide a means to manage secrets and data associated +with remote IT systems. +O.MANAGE_POLICY The TOE shall provide a workflow to manage authentication and access +control policies. +O.SEC_ACCESS The TOE shall ensure that only those authorized users and applications +are granted access to security functions and associated data. +O.PASSWD_PROT The TOE shall provide cryptographic mechanisms to protect passwords +via cryptographic processes including the ability to generate and destroy +keys. +O.TRANS_PROT The TOE shall provide mechanisms to protect data that is in transit +between elements within the TOE. +Table 11 ­ TOE Security Objectives +Security Objectives for the Operational Environment +The security objectives for the operational environment are addressed below: +OBJECTIVE DESCRIPTION +OE.TIME The TOE operating environment shall provide an accurate timestamp +(via reliable NTP server). +OE.ENV_PROTECT The TOE operating environment shall provide mechanisms to isolate the +TOE Security Functions (TSF) and assure that TSF components cannot +be tampered with or bypassed +OE.PERSONNEL Authorized administrators are non-hostile and follow all administrator +guidance and must ensure that the TOE is delivered, installed, managed, +and operated in a manner that maintains the TOE security objectives. +Any operator of the TOE must be trusted not to disclose their +authentication credentials to any individual not authorized for access to +the TOE. +OE.PHYSEC The facility surrounding the processing platform in which the TOE +resides must provide a controlled means of access into the facility +Table 12 ­ Operational Environment Security Objectives +Security Objectives Rationale + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 18 of 36 +This section provides the summary that all security objectives are traced back to aspects of the +addressed assumptions, threats, and Organizational Security Policies. +OBJECTIVES THREATS/ +ASSUMPTIONS/ POLICIES +O.MANAGE_DATA +O.MANAGE_POLICY +O.SEC_ACCESS +O.PASSWD_PROT +O.TRANS_PROT +OE.TIME +OE.ENV_PROTECT +OE.PERSONNEL +OE.PHYSEC +A.CONFIG +A.MANAGE +A.NOEVIL +A.LOCATE +A.TIMESOURCE +T.NO_AUTH +T.NO_PRIV +T.USER_ACCESS_DENY +T.PASSWD_COMPROMISE +T.PROT_TRANS +P. REMOTE_DATA +Table 13 ­ Mapping of Assumptions, Threats, Policies and ORSP s to Security Objectives +Mapping of Objectives +ASSUMPTION /THREAT/ +POLICY +RATIONALE +A.CONFIG This assumption is addressed by + OE.ENV_PROTECT, which ensures that TSF components +cannot be tampered with or bypassed + OE.PERSONNEL, which ensures that the TOE is managed +and administered by in a secure manner by a competent +and security aware personnel in accordance with the +administrator documentation. This objective also ensures +that those responsible for the TOE install, manage, and +operate the TOE in a secure manner + OE.PHYSEC, which ensures that the facility surrounding the +processing platform in which the TOE resides provides a +controlled means of access into the facility + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 19 of 36 +ASSUMPTION /THREAT/ +POLICY +RATIONALE +A.MANAGE This assumption is addressed by + OE.PERSONNEL, which ensures that the TOE is managed +and administered by in a secure manner by a competent +and security aware personnel in accordance with the +administrator documentation. This objective also ensures +that those responsible for the TOE install, manage, and +operate the TOE in a secure manner +A.NOEVIL This assumption is addressed by OE.PERSONNEL, which ensures +that the TOE is managed and administered by in a secure manner +by a competent and security aware personnel in accordance with +the administrator documentation. This objective also ensures +that those responsible for the TOE install, manage, and operate +the TOE in a secure manner +A.LOCATE This assumption is addressed by OE.PHYSEC which ensures that +the facility surrounding the processing platform in which the +TOE resides provides a controlled means of access into the +facility +A.TIMESOURCE This assumption is addressed by OE.TIME, which ensures the +provision of an accurate time source. +T.NO_AUTH This threat is countered by the following: + O.SEC_ACCESS, which ensures that the TOE allows access to +the security functions, configuration, and associated data +only by authorized users and applications +T.NO_PRIV This threat is countered by O.SEC_ACCESS, which ensures that +the TOE allows access to the security functions, configuration, +and associated data only by authorized users and applications. +T.PASSWD_COMPROMISE This threat is countered by O.PASSWD_PROT, which ensures +the passwords are not in the clear and cannot be exposed to un +authorized users for use. +T.PROT_TRANS This threat is countered by O.TRANS_PROT, which protects data +that is in transit between elements within the TOE. +P.REMOTE_DATA This organizational security policy is enforced by + O.MANAGE_DATA, which ensures that the TOE provide a +means to manage secrets and data associated with remote +IT systems. +T.USER_ACCESS_DENY This threat is countered by O.MANAGE_POLICY which ensures +that the TOE provides a workflow to manage authentication and +access control policies. +Table 14 ­ Mapping of Threats, Policies, and Assumptions to Objectives + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 20 of 36 +5. Extended Components Definition +This Security Target does include any extended components. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 21 of 36 +6. Security Requirements +The security requirements that are levied on the TOE and the IT environment are specified in +this section of the ST. +Security Functional Requirements +The functional security requirements for this Security Target consist of the following +components from Part 2 of the CC, which are summarized in the following table: +CLASS HEADING CLASS_FAMILY DESCRIPTION +Security Audit +FAU_GEN.1 Audit Data Generation +FAU_SAR.1 Audit Review +Cryptographic Support FCS_CKM.1 Cryptographic key generation +FCS_CKM.4 Cryptographic key destruction +FCS_COP.1 Cryptographic operation +User Data Protection +FDP_ACC.1 Subset Access Control +FDP_ACF.1 Security Attribute Based Access Control +Identification and +Authentication +FIA_ATD.1 User Attribute Definition +FIA_UID.2 User Identification before Any Action +FIA_UAU.2 User Authentication before Any Action +Security Management +FMT_MSA.1 Management of Security Attributes +FMT_MSA.2 Secure Security Attributes +FMT_MSA.3 Static Attribute Initialization +FMT_MTD.1 Management of TSF Data +FMT_SMF.1 Specification of Management Functions +FMT_SMR.1 Security Roles +Protection of the TSF FPT_TDC.1 Inter-TSF basic TSF data consistency +Trusted Path / Channels +FTP_ITC.1 Trusted Channel +FTP_TRP.1 Trusted Path +Table 15 ­ TOE Security Functional Requirements +Security Audit (FAU) +FAU_GEN.1 Audit Data Generation +FAU_GEN.1.1 The TSF shall be able to generate an audit record of the following +auditable events: +a) Start-up and shutdown of the audit functions; +b) All auditable events for the [not specified] level of audit; and +c) [User login/logout and; +d) Login failures;] + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 22 of 36 +FAU_GEN.1.2 The TSF shall record within each audit record at least the following +information: +a) Date and time of the event, type of event, subject identity (if +applicable), and the outcome (success or failure) of the event; +and +b) For each audit event type, based on the auditable event +definitions of the functional components included in the PP/ST, +[no other audit relevant information]. +FAU_SAR.1 Audit Review +FAU_SAR.1.1 The TSF shall provide [the Administrator] with the capability to read +[all audit data generated within the TOE] from the audit records. +FAU_SAR.1.2 The TSF shall provide the audit records in a manner suitable for the +user to interpret the information. +Cryptographic Support +FCS_CKM.1 Cryptographic key generation +FCS_CKM.1.1 The TSF shall generate cryptographic keys in accordance with a +specified cryptographic key generation algorithm [cryptographic key +generation algorithm in Table 16] and specified cryptographic key +sizes [cryptographic key sizes in Table 16] that meet the following: [list +of standards in Table 16]. +Usage Key Generation Algorithm Key Size (bits), Elliptical Curves Standard +RSA RSA Key Generation 2048 FIPS 186-4 +AES Deterministic Random Bit +Generator (DRBG) +128, 256 SP 800-90A +Diffie-Hellman Diffie-Hellman Key +Generation +1024, 2048 FIPS 186-4 +Table 16 ­ Cryptographic Standards +FCS_CKM.4 Cryptographic key destruction +FCS_CKM.4.1 The TSF shall destroy cryptographic keys in accordance with a +specified cryptographic key destruction method [zeroize] that meets +the following: [FIPS 140-2]. +FCS_COP.1 Cryptographic operation (Encryption / Decryption) +FCS_COP.1.1 The TSF shall perform [cryptographic operations in Table 17] in +accordance with a specified cryptographic algorithm [cryptographic +algorithm in Table 17] and cryptographic key sizes [cryptographic key +sizes in Table 17] that meet the following: [list of standards in Table +17]. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 23 of 36 +Application Note: AES in CBC mode is used for encrypting/decrypting +data in support of TLS. +Operation Algorithm Key Size, Curve or +Digest +Standard +Encryption and +Decryption in support of +TLS +AES (Advanced +Encryption +Standard) +128, 256 FIPS PUB +197 +Key agreement in +support TLS +Key Agreement +Schemes (KAS) and +Key Confirmation +P-256, P384, P521 SP800- +56A +Authentication algorithm +in support of TLS +ECDSA (Elliptic +Curve Digital +Signature +Algorithm) +P-256, P384, P521 FIPS 186-4 +Secure Hashing in +support of TLS +Secure Hash +Algorithm (SHA) +160 (SHA-1) +256 (SHA-256) +384 (SHA-384) +FIPS PUB +180-4 +Message Authentication +in support of TLS +Keyed-Hash +Message +Authentication Code +(HMAC) +160 (HMAC-SHA1) 256 +(HMAC-SHA2-256) 384 +(HMAC-SHA2-384) +FIPS 198-1 +Asymmetric +cryptography in support +of TLS +Rivest, Shamir, +Adleman (RSA) +2048 FIPS 186-4 +Table 17 ­ Cryptographic Operations +Information Flow Control (FDP) +FDP_ACC.1 Subset Access Control +FDP_ACC.1.1 The TSF shall enforce the [Discretionary Access Control SFP] on [ +Subjects: All users +Objects: System reports, component audit logs, TOE configuration, +operator account attributes +Operations: all user actions] +FDP_ACF.1 Security Attribute Based Access Control +FDP_ACF.1.1 The TSF shall enforce the [Discretionary Access Control SFP]to objects +based on the following: [ +Subjects: All users +Objects: System reports, component audit logs, TOE configuration, +operator account attributes +Operations: all user actions] +FDP_ACF.1.2 The TSF shall enforce the following rules to determine if an operation +among controlled subjects and controlled objects is allowed: [if the + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 24 of 36 +ACL identifies the user or a group of users that contains the user +requesting access for the type of resource that the user is requesting, +and the user (or group of users) has the specific rights required for the +type of operation requested on the object then the user is granted +access]. +FDP_ACF.1.3 The TSF shall explicitly authorize access of subjects to objects based +on the following additional rules: [password restrictions, login +restrictions, time based access controls, ip access controls, intruder +lockout]. +FDP_ACF.1.4 The TSF shall explicitly deny access of subjects to objects based on the +following additional rules [ password restrictions, login restrictions, +time based access controls, ip access controls, intruder lockout] +Identification and Authentication (FIA) +FIA_ATD.1 ­ User Attribute Definition +FIA_ATD.1.1 The TSF shall maintain the following list of security attributes +belonging to individual users: [User Identity, Authentication Status, +and Privilege Level]. +FIA_UAU.2 User Authentication before Any Action +FIA_UAU.2.1 The TSF shall require each user to be successfully authenticated +before allowing any other TSF-mediated actions on behalf of that user. +FIA_UID.2 User Identification before Any Action +FIA_UID.2.1 The TSF shall require each user to be successfully identified before +allowing any other TSF-mediated actions on behalf of that user. +Security Management (FMT) +FMT_MSA.1 Management of security attributes +FMT_MSA.1.1 The TSF shall enforce the [Discretionary Access Control SFP] to +restrict the ability to [query, modify, delete] the security attributes +[Accounts, privileges, ACLs] to [Administrator]. +FMT_MSA.2 Secure Security Attributes +FMT_MSA.2.1 The TSF shall ensure that only secure values are accepted for +[security attributes listed with Discretionary Access Control SFP]. +FMT_MSA.3 Static Attribute Initialization +FMT_MSA.3.1 The TSF shall enforce the [Discretionary Access Control SFP] to +provide [restrictive] default values for security attributes that are +used to enforce the SFP. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 25 of 36 +FMT_MSA.3.2 The TSF shall allow the [Administrator] to specify alternative initial +values to override the default values when an object or information +is created. +FMT_MTD.1 Management of TSF Data +FMT_MTD.1.1 The TSF shall restrict the ability to [control] the [data described in the +table below] to [Administrator]: +DATA CHANGE QUERY MODIFY DELETE CLEAR +Discretionary +Access Control SFP + +User Account +Attributes + +Audit Logs +Date/Time +Table 18 ­ Management of TSF data +FMT_SMF.1 Specification of Management Functions +FMT_SMF.1.1 The TSF shall be capable of performing the following management +functions: [ +a) Create accounts +b) Modify accounts +c) Define privilege levels Change Default, +Query, Modify, Delete, Clear the attributes +associated with the Discretionary Access +Control SFP +d) Modify the behavior of the Discretionary +Access Control SFP +e) Manage ACLs]. +FMT_SMR.1 Security Roles +FMT_SMR.1.1 The TSF shall maintain the roles [Administrator, User]. +FMT_SMR.1.2 The TSF shall be able to associate users with roles. +Protection of the TSF (FPT) +FPT_TDC.1 Inter-TSF Basic TSF Data Consistency +FPT_TDC.1.1 The TSF shall provide the capability to consistently interpret [secrets +(passwords)] when shared between the TSF and another trusted IT +product. +FPT_TDC.1.2 The TSF shall use [the secret with the newest associated timestamp] +when interpreting the TSF data from another trusted IT product. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 26 of 36 +Trusted Path / Channel (FTP) +FTP_ITC.1 Inter-TSF trusted channel +FTP_ITC.1.1 The TSF shall provide a communication channel between itself and +[another trusted IT product] that is logically distinct from other +communication channels and provides assured identification of its end +points and protection of the channel data from [modification or +disclosure]. +FTP_ITC.1.2 The TSF shall permit [the TSF] to initiate communication via the +trusted channel. +FTP_ITC.1.3 The TSF shall initiate communication via the trusted channel for +[HTTPS/TLS connections + for communications labeled 1 ­ 12 in Figure 1] +Application Note: The TOE supports TLS v1.1 and 1.2 as configured by +the Administrator. +Application Note: Crypto as claimed in FCS_COP_1 is used to support +TLS. +FTP_TRP.1 Trusted Path +FTP_TRP.1.1 The TSF shall provide a communication path between itself and [local] +users that is logically distinct from other communication paths and +provides assured identification of its end points and protection of the +communicated data from [disclosure]. +FTP_TRP.1.2 The TSF shall permit [the TSF] to initiate communication via the +trusted path. +FTP_TRP.1.3 The TSF shall require the use of the trusted path for [key requests, and +encryption operations + for communications labeled A, B, and C in Figure 1] +Security Assurance Requirements +The Security Assurance Requirements for this evaluation are listed in Section 6.3.4 ­ Security +Assurance Requirements. +Security Requirements Rationale +Security Functional Requirements +The following table provides the correspondence mapping between security objectives and the +requirements that satisfy them. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 27 of 36 +OBJECTIVE +SFR +O.MANAGE_DATA +O.MANAGE_POLICY +O.SEC_ACCESS +O.PASSWD_PROT +O.TRANS_PROT +FAU_GEN.1 +FAU_SAR.1 +FCS_CKM.1 +FCS_CKM.4 +FCS_COP.1 +FDP_ACC.1 +FDP_ACF.1 +FIA_ATD.1 +FIA_UID.2 +FIA_UAU.2 +FMT_MSA.1 +FMT_MSA.2 +FMT_MSA.3 +FMT_MTD.1 +FMT_SMF.1 +FMT_SMR.1 +FPT_TDC.1 +FTP_ITC.1 +FTP_TRP.1 +Table 19 ­ Mapping of TOE Security Functional Requirements and Objectives +Dependency Rationale +This ST satisfies all the security functional requirement dependencies of the Common Criteria. +The table below lists each SFR to which the TOE claims conformance with a dependency and +indicates whether the dependent requirement was included. As the table indicates, all +dependencies have been met. +SFR CLAIM DEPENDENCIES DEPENDENCY MET RATIONALE +FAU_GEN.1 FPT_STM.1 YES +Satisfied by the Operational +Environment (OE.TIME) +FAU_SAR.1 +FAU_GEN.1 +FPT_STM.1 +YES +FPT_STM.1 satisfied by the +Operational Environment +(OE.TIME) + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 28 of 36 +SFR CLAIM DEPENDENCIES DEPENDENCY MET RATIONALE +FCS_CKM.1 +FCS_CKM.1 or +FCS_COP.1 and +FCS_CKM.4 +YES +Satisfied by FCS_COP.1 and +FCS_CKM.4 +FCS_CKM.4 FTP_ITC.1 or +FTP_ITC.2 or +FCS_CKM.1 +YES Satisfied by FCS_CKM.1 for AES +FCS_COP.1 FTP_ITC.1 or +FTP_ITC.2 or +FCS_CKM.1 and +FCS_CKM.4 +YES Satisfied by FCS_CKM.1 and +FCS_CKM.4 +FDP_ACC.1 FDP_ACF.1 YES +FDP_ACF.1 +FDP_ACC.1 +FMT_MSA.3 +YES +FIA_ATD.1 N/A N/A +FIA_UID.2 N/A N/A +FMT_MSA.1 +FDP_ACC.1 +FMT_SMF.1 +FMT_SMR.1 +YES +FMT_MSA.2 +FDP_ACC.1 +FMT_MSA.1 +FMT_SMR.1 +YES +FMT_MSA.3 +FMT_MSA.1 +FMT_SMR.1 +YES +FMT_MTD.1 +FMT_SMF.1 +FMT_SMR.1 +YES +FMT_SMF.1 N/A N/A +FMT_SMR.1 FIA_UID.1 YES +Although FIA_UID.1 is not +included, FIA_UID.2, which is +hierarchical to FIA_UID.1 is +included. This satisfies this +dependency. +FPT_TDC.1 N/A N/A +FTP_ITC.1 N/A N/A +FTP_TRP.1 N/A N/A +Table 20 ­ Mapping of SFR to Dependencies and Rationales +Sufficiency of Security Requirements +The following table presents a mapping of the rationale of TOE Security Requirements to +Objectives. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 29 of 36 +OBJECTIVE RATIONALE +O.MANAGE_DATA The objective to ensure that the TOE will collect events from security +products and non-security products deployed within a network and +applies analytical processes to derive conclusions about the events is +met by the following security requirements: + FPT_TDC.1 ensures that the TOE provides consistency between +passwords used on remote IT systems and those +stored/managed within the TOE. +O.MANAGE_POLICY The objective to ensure that the TOE provides a workflow to manage +authentication and access control policies is met by the following +security requirements: + FAU_GEN.1 and FAU_SAR.1 define the auditing capability for +incidents and administrative access control and requires that +authorized users will have the capability to read and interpret +data stored in the audit logs + FMT_SMF.1 and FMT_SMR.1 support the security functions +relevant to the TOE and ensure the definition of an authorized +administrator role +O.SEC_ACCESS This objective ensures that the TOE allows access to the security +functions, configuration, and associated data only by authorized users +and applications. + FDP_ACC.1 requires that all user actions resulting in the access +to TOE security functions and configuration data are controlled + FDP_ACF.1 supports FDP_ACC.1 by ensuring that access to TOE +security functions, configuration data, audit logs, and account +attributes is based on the user privilege level and their +allowable actions + FIA_UID.2 requires the TOE to enforce identification of all users +prior to configuration of the TOE + FIA_UAU.2 requires the TOE to enforce authentication of all +users prior to configuration of the TOE + FIA_ATD.1 specifies security attributes for users of the TOE + FMT_MTD.1 restricts the ability to query, add or modify TSF +data to authorized users. + FMT_MSA.1 specifies that only privileged administrators can +access the TOE security functions and related configuration +data. + FMT_MSA.2 specifies that only secure values are accepted for +security attributes listed with access control policies. + FMT_MSA.3 ensures that the default values of security +attributes are restrictive in nature as to enforce the access +control policy for the TOE + FTP_ITC.1 specifies that the trusted channel exists for components +HTTPS/TLS. + FTP_TRP.1 specifies that the trusted path exists for components +HTTPS/TLS. + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 30 of 36 +OBJECTIVE RATIONALE +O.PASSWD_PROT This objective ensures that the TOE provides cryptographic +mechanisms to generate and destroy keys. This objective is met by: +FCS_CKM.1, FCS_CKM. 4, and FCS_COP.1 which provide the +cryptographic support functions for secure communications within the +TOE and with external IT entities. +O.TRANS_PROT This objective ensures that the TOE protects data in transit between +elements within the TOE. This objective is met by FTP_ITC (which +specifies that the trusted channel exists for components) and FTP_TRP +(which ensures that the trusted path exists for components). +Table 21 ­ Rationale for TOE SFRs to Objectives +Security Assurance Requirements +The assurance security requirements for this Security Target are taken from Part 3 of the CC. +These assurance requirements compose an Evaluation Assurance Level 3 (EAL3). The assurance +components are summarized in the following table: +CLASS HEADING CLASS_FAMILY DESCRIPTION +ADV: Development +ADV_ARC.1 Security Architecture Description +ADV_FSP.3 +Functional Specification with Complete +Summary +ADV_TDS.2 Architectural Design +AGD: Guidance +Documents +AGD_OPE.1 Operational User Guidance +AGD_PRE.1 Preparative Procedures +ALC: Lifecycle Support +ALC_CMC.3 Authorization Controls +ALC_CMS.3 Implementation representation CM coverage +ALC_DEL.1 Delivery Procedures +ALC_DVS.1 Identification of Security Measures +ALC_LCD.1 Developer defined life-cycle model +ALC_FLR.2 Flaw Reporting Procedures +ATE: Tests +ATE_COV.2 Analysis of Coverage +ATE_DPT.1 Testing: Basic Design +ATE_FUN.1 Functional Testing +ATE_IND.2 Independent Testing - Sample +AVA: Vulnerability +Assessment +AVA_VAN.2 Vulnerability Analysis +Table 22 ­ Security Assurance Requirements at EAL3 +Security Assurance Requirements Rationale +The ST specifies Evaluation Assurance Level 3. EAL3 was chosen because it is based upon good +commercial development practices with thorough functional testing. EAL3 provides the +developers and users a moderate level of independently assured security in conventional +commercial TOEs. The threat of malicious attacks is not greater than low, the security +environment provides physical protection, and the TOE itself offers a very limited interface, + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 31 of 36 +offering essentially no opportunity for an attacker to subvert the security policies without +physical access. The product was augmented to comply with ALC_FLR.2 in order to document +and address requirements for remediation and reporting of faults that may be discovered in the +product after release. +Security Assurance Requirements Evidence +This section identifies the measures applied to satisfy CC assurance requirements. +SECURITY ASSURANCE +REQUIREMENT +EVIDENCE TITLE +ADV_ARC.1 Security Architecture +Description +NetIQ Identity Manager 4.7 +Security Architecture (ADV_ARC) +ADV_FSP.3 Functional Specification +with Complete Summary +NetIQ Identity Manager 4.7 +Functional Specification (ADV_FSP) +ADV_TDS.2 Architectural Design +NetIQ Identity Manager 4.7 +Architectural Design (IDM TDS) +AGD_OPE.1 Operational User +Guidance4 +NetIQ Identity Manager 4.7 +Operational User Guidance and Preparative +Procedures Supplement (AGD-IGS) +AGD_PRE.1Preparative Procedures +NetIQ Identity Manager 4.7 +Operational User Guidance and Preparative +Procedures Supplement (AGD-IGS) +ALC_CMC.3 Authorization Controls +NetIQ Identity Manager 4.7 +Configuration Management Processes and +Procedures (ALC_CM) +ALC_CMS.3 Implementation +representation CM coverage +NetIQ Identity Manager 4.7 +Configuration Management Processes and +Procedures (ALC_CM) +ALC_DEL.1 Delivery Procedures +NetIQ Identity Manager 4.7 +Secure Delivery Processes and Procedures +(ALC_DEL) +ALC_DVS.1 Identification of Security +Measures +NetIQ Identity Manager 4.7 +Development Security Measures (ALC_DVS) +ALC_LCD.1 Developer defined life- +cycle model +NetIQ Identity Manager 4.7 +Life Cycle Development Process (ALC_LCD) +ALC_FLR.2: Flaw Remediation +Procedures +NetIQ Identity Manager 4.7 +Flaw reporting Procedures (ALC_FLR) +ATE_COV.2 Analysis of Coverage +NetIQ Identity Manager 4.7 +Test Plan and Coverage Analysis (ATE) +ATE_DPT.1 Testing: Basic Design +NetIQ Identity Manager 4.7 +Test Plan and Coverage Analysis (ATE) +4 +Additional documents can be found in Appendix A + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 32 of 36 +SECURITY ASSURANCE +REQUIREMENT +EVIDENCE TITLE +ATE_FUN.1Functional Testing +NetIQ Identity Manager 4.7 +Test Plan and Coverage Analysis (ATE) +Table 23 ­ Security Assurance Rationale and Measures + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 33 of 36 +7. TOE Summary Specification +This section presents the Security Functions implemented by the TOE. +TOE Security Functions +The security functions performed by the TOE are as follows: + Security Management + Security Audit + Identification and Authentication + User Data Protection + Trusted Path / Channels + Cryptographic Support +Security Audit +The TOE generates the following audit data: + Start-up and shutdown of the audit functions (instantiated by startup of the TOE) + User login/logout + Login failures +The TOE provides the Administrator with the capability to read all audit data generated within +the TOE via the console. The GUI provides a suitable means for an Administrator to interpret the +information from the audit log. +The A.TIMESOURCE is added to the assumptions on operational environment, and OE.TIME is +added to the operational environment security objectives. The time and date provided by the +operational environment are used to form the timestamps. The TOE ensures that the audit trail +data is stamped when recorded with a dependable date and time received from the OE +(operating system). In this manner, accurate time and date is maintained on the TOE. +The Security Audit function is designed to satisfy the following security functional requirements: + FAU_GEN.1 + FAU_SAR.1 +Identification and Authentication +The IDM console application provides user interfaces that administrators may use to manage +TOE functions. The operating system and the database in the TOE Environment are queried to +individually authenticate administrators or users. The TOE maintains authorization information +that determines which TOE functions an authenticated administrators or users (of a given role) +may perform. +The TOE maintains the following list of security attributes belonging to individual users: + User Identity (i.e., user name) + Authentication Status (whether the IT Environment validated the username/password) + Privilege Level (Administrator or User) +The Identification and Authentication function is designed to satisfy the following security +functional requirements: + FIA_ATD.1 + FIA_UAU.2 + FIA_UID.2 +User Data Protection + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 34 of 36 +The TOE implements a discretionary access control policy to define what roles can access +particular functions of the TOE. All access and actions for system reports, component audit logs, +TOE configuration, operator account attributes (defined in FIA_ATD.1) are protected via access +control list. When a user requests to perform an action on an object, the TOE verifies the role +associated with the user name. Access is granted if the user (or group of users) has the specific +rights required for the type of operation requested on the object. +Identity Manager can enforce password policies on incoming passwords from connected +systems and on passwords set or changed through the User Application password self-service. +If the new password does not comply, you can specify that Identity Manager not accept the +password. This also means that passwords that don't comply with your policies are not +distributed to other connected systems. +In addition, can enforce password policies on connected systems. If the password being +published to the Identity Vault does not comply with rules in a policy, you can specify that +Identity Manager not only does not accept the password for distribution, but actually resets the +noncompliant password on the connected system by using the current Distribution password in +the Identity Vault. +The User Data Protection function is designed to satisfy the following security functional +requirements: + FDP_ACC.1 + FDP_ACF.1 + FPT_TDC.1 +Security Management +The TOE maintains the operator roles described in the following table. The individual roles are +categorized into two main roles: the Administrator and the User. +ROLE MANAGEMENT FUNCTIONS +Administrator A user who has rights to configure and manage all aspects of the TOE +User The user's capabilities can be configured to: +View hierarchical relationships between User objects +View and edit user information (with appropriate rights). +Search for users or resources using advanced search criteria +(which can be saved for later reuse). +Recover forgotten passwords. +Table 24 ­ Roles and Functions +Only an Administrator can determine the behavior of, disable, enable, and modify the behavior +of the functions that implement the Discretionary Access Control SFP. The TPE ensures only +secure values are accepted for the security attributes listed with Discretionary Access Control +SFP. +The Security Management function is designed to satisfy the following security functional +requirements: + FMT_MTD.1 + FMT_MSA.1 + FMT_MSA.2 + FMT_MSA.3 + FMT_SMF.1 + FMT_SMR.1 + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 35 of 36 +Trusted Path / Channels +The Trusted Path/Channels function is designed to satisfy the following security functional +requirements: + FTP_ITC.1 ­ the TOE supports establishment of trusted channels for communicating +TOE entities using HTTPS. + FTP_TRP.1 ­ the TOE provides a trusted path for TOE Users, using HTTPS +Trusted Channel +The TOE provides a trusted channel between the TOE and external web servers. +Trusted channels are implemented using HTTPS. The TOE supports TLS v1.1 and TLS v1.2. The +TOE supports the following TLS cipher suites, as defined in RFC 2246, RFC 4346 and RFC 5246: + TLS_RSA_WITH_AES_128_CBC_SHA + TLS_RSA_WITH_AES_128_GCM_SHA256 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + TLS_ECDHE_ECDSA_WITH_AES_256_ CBC_SHA + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +Trusted Path: +The TOE provides a trusted path for TOE administrators and TOE users to communicate with +the TOE. The trusted path is implemented using HTTPS. The TOE's implementation of TLS is +described in the previous section (Trusted Channel). +Cryptographic Support +Cryptographic protection of data in transit between the TOE and remote users, and between +the TOE and external web servers is provided by the OpenSSL FIPS Object Module software +version 2.0.10 (Cryptographic Module Validation Program (CMVP) certificate number 1747) +libraries. +The following table identifies the CAVP algorithm certificates. +Operation Algorithm CAVP Certificate +Encryption and Decryption in +support of TLS +AES (Advanced Encryption +Standard) +AES 3264 +Key Generation in support of +TLS +DRBG (Deterministic +Random Bit Generation) +DRBG 723 +Key agreement in support of +TLS +Key Agreement Schemes +(KAS) and Key Confirmation +CVL 472 +Keyed-Hash Message +Authentication in support of +TLS +HMAC-SHA1, HMAC-SHA2- +256, HMAC-SHA2-384 +HMAC 2063 +Secure Hash in support of TLS SHA-1, SHA-256, SHA-384 SHS 2702 + June 1, 2020 NetIQ Identity Manager 4.7 ST +NetIQ Corporation Page 36 of 36 +Asymmetric cryptography in +support of TLS +RSA RSA 1664 +Authentication algorithm in +support of TLS +ECDSA ECDSA 620 +Table 25 ­ CAVP +The Cryptographic Support function is designed to satisfy the following security functional +requirements: + FCS_CKM.1 + FCS_CKM.4 + FCS_COP.1 + \ No newline at end of file diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py index f6471821..c643444b 100644 --- a/test/test_cc_oop.py +++ b/test/test_cc_oop.py @@ -72,6 +72,9 @@ class TestCommonCriteriaOOP(TestCase): self.template_target_pdf_hashes = {'869415cc4b91282e': 'b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9', '2d010ecfb604747a': '3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be'} + self.template_report_txt_path = self.test_data_dir / 'report_869415cc4b91282e.txt' + self.template_target_txt_path = self.test_data_dir / 'target_869415cc4b91282e.txt' + def test_certificate_input_sanity(self): self.assertEqual(self.crt_one.report_link, 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf', @@ -94,7 +97,7 @@ class TestCommonCriteriaOOP(TestCase): new_obj = json.load(handle, cls=CustomJSONDecoder) return obj == new_obj - def test_download_pdfs(self): + def test_download_and_convert_pdfs(self): with open(self.test_data_dir / 'toy_dataset.json', 'r') as handle: dset = json.load(handle, cls=CustomJSONDecoder) @@ -102,6 +105,7 @@ class TestCommonCriteriaOOP(TestCase): dset.root_dir = Path(td) dset.download_all_pdfs() + dset.convert_all_pdfs() actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.report_pdf_paths.items()} actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.target_pdf_paths.items()} @@ -109,6 +113,12 @@ class TestCommonCriteriaOOP(TestCase): self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (certificate report) do not the template') self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template') + self.assertTrue(filecmp.cmp(dset.report_txt_paths['869415cc4b91282e'], self.template_report_txt_path), + 'The report of 869415cc4b91282e.pdf converted to txt does not match the template.') + self.assertTrue(filecmp.cmp(dset.target_txt_paths['869415cc4b91282e'], self.template_target_txt_path), + 'The target of 869415cc4b91282e.pdf converted to txt does not match the template.') + + def test_cert_to_json(self): self.assertTrue(self.equal_to_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), 'The certificate serialized to json differs from a template.') -- cgit v1.3.1 From 83c22746cb6f8bb7f5d5d90bb6b3155529c0c674 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 Nov 2020 11:14:19 +0100 Subject: Fix cc_oop_test of pdf file conversion It seems that pdftotext is not fully reproducible and gives platform specific results. So we dont check for full file equality, but rather for the existence of converted file. We further check that the created file size roughly matches the one freshly produced by Travis pdftotext. --- test/test_cc_oop.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py index c643444b..4ad9b320 100644 --- a/test/test_cc_oop.py +++ b/test/test_cc_oop.py @@ -113,11 +113,12 @@ class TestCommonCriteriaOOP(TestCase): self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (certificate report) do not the template') self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template') - self.assertTrue(filecmp.cmp(dset.report_txt_paths['869415cc4b91282e'], self.template_report_txt_path), - 'The report of 869415cc4b91282e.pdf converted to txt does not match the template.') - self.assertTrue(filecmp.cmp(dset.target_txt_paths['869415cc4b91282e'], self.template_target_txt_path), - 'The target of 869415cc4b91282e.pdf converted to txt does not match the template.') - + self.assertTrue(dset.report_txt_paths['869415cc4b91282e'].exists()) + self.assertTrue(dset.target_txt_paths['869415cc4b91282e'].exists()) + self.assertAlmostEqual(dset.target_txt_paths['869415cc4b91282e'].stat().st_size, + self.template_target_txt_path.stat().st_size, delta=1000) + self.assertAlmostEqual(dset.report_txt_paths['869415cc4b91282e'].stat().st_size, + self.template_report_txt_path.stat().st_size, delta=1000) def test_cert_to_json(self): self.assertTrue(self.equal_to_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), -- cgit v1.3.1 From 5fa687185a3008b5f0ef4ccda7ece38f57012306 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 Nov 2020 13:29:28 +0100 Subject: Improve (de)serialization of objects from/in json - Every complex serialiazable object now inherits from `ComplexSerializableType` - All such objects now implement `to_dict()`, `from_dict()` methods - Some objects for which standalone json file makes sense implement `to_json()`, `from_json()` - Rewrite test and demos - Datasets `root_dir` is now not part of serialization, but only internal representation --- cc_oop_demo.py | 7 ++---- fips_oop_demo.py | 2 +- sec_certs/certificate.py | 19 +++++++++++---- sec_certs/dataset.py | 29 ++++++++++++++--------- sec_certs/serialization.py | 30 +++++++++++++++++------- test/data/test_cc_oop/toy_dataset.json | 1 - test/test_cc_oop.py | 43 ++++++++++++++-------------------- 7 files changed, 74 insertions(+), 57 deletions(-) diff --git a/cc_oop_demo.py b/cc_oop_demo.py index 54e07307..c78759df 100644 --- a/cc_oop_demo.py +++ b/cc_oop_demo.py @@ -26,13 +26,10 @@ def main(): logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') # Dump dataset into JSON - with open('./debug_dataset/cc_full_dataset.json', 'w') as handle: - json.dump(dset, handle, cls=CustomJSONEncoder, indent=4) + dset.to_json('./debug_dataset/cc_full_dataset.json') # Load dataset from JSON - with open('./debug_dataset/cc_full_dataset.json', 'r') as handle: - new_dset = json.load(handle, cls=CustomJSONDecoder) - new_dset.root_dir = Path('/Users/adam/phd/projects/certificates/sec-certs/debug_dataset') + new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json') assert dset == new_dset diff --git a/fips_oop_demo.py b/fips_oop_demo.py index 6f556394..9e283670 100644 --- a/fips_oop_demo.py +++ b/fips_oop_demo.py @@ -16,7 +16,7 @@ def main(): logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') # Dump dataset into JSON - dset.dump_to_json() + dset.to_json(dset.root_dir / 'fips_full_dataset.json') logging.info(f'Dataset saved to {dset.root_dir}/fips_full_dataset.json') logging.info("Extracting keywords now.") diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index d2585188..724a60e0 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -5,12 +5,14 @@ import logging from pathlib import Path import os import copy +import json from abc import ABC, abstractmethod from bs4 import Tag, BeautifulSoup, NavigableString from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type from sec_certs import helpers, extract_certificates +from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder logger = logging.getLogger(__name__) @@ -42,8 +44,17 @@ class Certificate(ABC): def from_dict(cls: Type[T], dct: dict) -> T: return cls(*tuple(dct.values())) + def to_json(self, output_path: Union[Path, str]): + with Path(output_path).open('w') as handle: + json.dump(self, handle, indent=4, cls=CustomJSONEncoder) -class FIPSCertificate(Certificate): + @classmethod + def from_json(cls, input_path: Union[Path, str]): + with Path(input_path).open('r') as handle: + return json.load(handle, cls=CustomJSONDecoder) + + +class FIPSCertificate(Certificate, ComplexSerializableType): FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov' FIPS_MODULE_URL: ClassVar[ str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/' @@ -323,12 +334,12 @@ class FIPSCertificate(Certificate): []) -class CommonCriteriaCert(Certificate): +class CommonCriteriaCert(Certificate, ComplexSerializableType): cc_url = 'http://www.commoncriteriaportal.org' empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/' @dataclass(eq=True, frozen=True) - class MaintainanceReport: + class MaintainanceReport(ComplexSerializableType): """ Object for holding maintainance reports. """ @@ -357,7 +368,7 @@ class CommonCriteriaCert(Certificate): return self.maintainance_date < other.maintainance_date @dataclass(eq=True, frozen=True) - class ProtectionProfile: + class ProtectionProfile(ComplexSerializableType): """ Object for holding protection profiles. """ diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 0bfc7fb0..85b95b7c 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -3,7 +3,7 @@ import re from datetime import datetime import locale import logging -from typing import Dict, List, ClassVar, Collection, TypeVar, Type +from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union import json from importlib import import_module @@ -29,6 +29,7 @@ from sec_certs.constants import FIPS_NOT_AVAILABLE_CERT_SIZE import sec_certs.constants as constants import sec_certs.download as download import sec_certs.cert_processing as cert_processing +from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder logger = logging.getLogger(__name__) @@ -63,17 +64,28 @@ class Dataset(ABC): return str(type(self).__name__) + ':' + self.name + ', ' + str(len(self)) + ' certificates' def to_dict(self): - return {'root_dir': copy.deepcopy(self.root_dir), 'timestamp': self.timestamp, - 'sha256_digest': self.sha256_digest, 'name': self.name, 'description': self.description, + return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest, + 'name': self.name, 'description': self.description, 'n_certs': len(self), 'certs': list(self.certs.values())} @classmethod def from_dict(cls, dct: Dict): certs = {x.dgst: x for x in dct['certs']} - dset = cls(certs, Path(dct['root_dir']), dct['name'], dct['description']) + dset = cls(certs, Path('./'), dct['name'], dct['description']) assert len(dset) == dct['n_certs'] return dset + def to_json(self, output_path: Union[str, Path]): + with Path(output_path).open('w') as handle: + json.dump(self, handle, indent=4, cls=CustomJSONEncoder) + + @classmethod + def from_json(cls, input_path: Union[str, Path]): + with Path(input_path).open('r') as handle: + dset = json.load(handle, cls=CustomJSONDecoder) + dset.root_path = input_path.parent + return dset + @abstractmethod def get_certs_from_web(self): raise NotImplementedError('Not meant to be implemented by the base class.') @@ -122,7 +134,7 @@ class Dataset(ABC): # TODO: Delete -class CCDataset(Dataset): +class CCDataset(Dataset, ComplexSerializableType): def __init__(self, certs: Dict[str, 'CommonCriteriaCert'], root_dir: Path, name: str = 'dataset name', description: str = 'dataset_description'): super().__init__(certs, root_dir, name, description) @@ -442,7 +454,7 @@ class CCDataset(Dataset): self._convert_targets_to_txt() -class FIPSDataset(Dataset): +class FIPSDataset(Dataset, ComplexSerializableType): FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov' FIPS_MODULE_URL: ClassVar[ str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/' @@ -490,11 +502,6 @@ class FIPSDataset(Dataset): self.keywords = json.loads( open(self.root_dir / 'fips_full_keywords.json').read()) - def dump_to_json(self): - with open(self.root_dir / 'fips_full_dataset.json', 'w') as handle: - json.dump(self, handle, cls=import_module( - 'sec_certs.serialization').CustomJSONEncoder, indent=4) - def dump_keywords(self): with open(self.root_dir / "fips_full_keywords.json", 'w') as f: f.write(json.dumps(self.keywords, indent=4, sort_keys=True)) diff --git a/sec_certs/serialization.py b/sec_certs/serialization.py index 15c38a7b..d37dd067 100644 --- a/sec_certs/serialization.py +++ b/sec_certs/serialization.py @@ -2,18 +2,24 @@ import json from datetime import date from pathlib import Path -from sec_certs.dataset import CCDataset, FIPSDataset -from sec_certs.certificate import CommonCriteriaCert, FIPSCertificate +from abc import ABC, abstractmethod -serializable_complex_types = ( -CCDataset, FIPSDataset, CommonCriteriaCert, CommonCriteriaCert.MaintainanceReport, CommonCriteriaCert.ProtectionProfile, -FIPSCertificate) -serializable_complex_types_dict = {x.__name__: x for x in serializable_complex_types} + +class ComplexSerializableType(ABC): + @classmethod + @abstractmethod + def to_dict(cls): + raise NotImplementedError + + @classmethod + @abstractmethod + def from_dict(cls): + raise NotImplementedError class CustomJSONEncoder(json.JSONEncoder): def default(self, obj): - if isinstance(obj, serializable_complex_types): + if isinstance(obj, ComplexSerializableType): return {**{'_type': type(obj).__name__}, **obj.to_dict()} if isinstance(obj, set): return sorted(list(obj)) @@ -25,12 +31,18 @@ class CustomJSONEncoder(json.JSONEncoder): class CustomJSONDecoder(json.JSONDecoder): + """ + Custom JSONDecoder. Any complex object that should be de-serializable must inherit directly from class + ComplexSerializableType (nested inheritance does not currently work (because x.__subclassess__() prints only direct + subclasses. Any such class must implement methods to_dict() and from_dict(). These are used to drive serialization. + """ def __init__(self, *args, **kwargs): json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) + self.serializable_complex_types = {x.__name__: x for x in ComplexSerializableType.__subclasses__()} def object_hook(self, obj): - if '_type' in obj and obj['_type'] in serializable_complex_types_dict.keys(): + if '_type' in obj and obj['_type'] in self.serializable_complex_types.keys(): complex_type = obj.pop('_type') - return serializable_complex_types_dict[complex_type].from_dict(obj) + return self.serializable_complex_types[complex_type].from_dict(obj) return obj diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json index 24392ad8..a16be012 100644 --- a/test/data/test_cc_oop/toy_dataset.json +++ b/test/data/test_cc_oop/toy_dataset.json @@ -1,6 +1,5 @@ { "_type": "CCDataset", - "root_dir": "/fictional/path/to/dataset", "timestamp": "2020-11-16 17:04:14.770153", "sha256_digest": "not implemented", "name": "toy dataset", diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py index 4ad9b320..98c11a4e 100644 --- a/test/test_cc_oop.py +++ b/test/test_cc_oop.py @@ -1,6 +1,6 @@ from unittest import TestCase from pathlib import Path -from tempfile import TemporaryDirectory, mkstemp +from tempfile import TemporaryDirectory, mkstemp, NamedTemporaryFile from datetime import date, datetime import json import filecmp @@ -80,23 +80,6 @@ class TestCommonCriteriaOOP(TestCase): 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf', 'Report link contains some improperly escaped characters.') - @staticmethod - def equal_to_json(referential_path, obj): - fd, path = mkstemp() - try: - with os.fdopen(fd, 'w') as handle: - json.dump(obj, handle, cls=CustomJSONEncoder, indent=4) - - return filecmp.cmp(referential_path, path) - finally: - os.remove(path) - - @staticmethod - def equal_from_json(referential_path, obj): - with open(referential_path, 'r') as handle: - new_obj = json.load(handle, cls=CustomJSONDecoder) - return obj == new_obj - def test_download_and_convert_pdfs(self): with open(self.test_data_dir / 'toy_dataset.json', 'r') as handle: dset = json.load(handle, cls=CustomJSONDecoder) @@ -121,20 +104,28 @@ class TestCommonCriteriaOOP(TestCase): self.template_report_txt_path.stat().st_size, delta=1000) def test_cert_to_json(self): - self.assertTrue(self.equal_to_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), - 'The certificate serialized to json differs from a template.') + with NamedTemporaryFile('w') as tmp: + self.fictional_cert.to_json(tmp.name) + self.assertTrue(filecmp.cmp(self.test_data_dir / 'fictional_cert.json', + tmp.name), + 'The certificate serialized to json differs from a template.') def test_dataset_to_json(self): - self.assertTrue(self.equal_to_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset), - 'The dataset serialized to json differs from a template.') + with NamedTemporaryFile('w') as tmp: + self.template_dataset.to_json(tmp.name) + self.assertTrue(filecmp.cmp(self.test_data_dir / 'toy_dataset.json', + tmp.name), + 'The dataset serialized to json differs from a template.') def test_cert_from_json(self): - self.assertTrue(self.equal_from_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert), - 'The certificate serialized from json differs from a template.') + self.assertEqual(self.fictional_cert, + CommonCriteriaCert.from_json(self.test_data_dir / 'fictional_cert.json'), + 'The certificate serialized from json differs from a template.') def test_dataset_from_json(self): - self.assertTrue(self.equal_from_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset), - 'The dataset serialized from json differs from a template.') + self.assertEqual(self.template_dataset, + CCDataset.from_json(self.test_data_dir / 'toy_dataset.json'), + 'The dataset serialized from json differs from a template.') def test_build_empty_dataset(self): with TemporaryDirectory() as tmp_dir: -- cgit v1.3.1 From 2eeabc1d6d8cff35aff7ade1932596c104eb3d4d Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 Nov 2020 14:16:44 +0100 Subject: type hinting checks --- sec_certs/cert_processing.py | 6 ++++-- sec_certs/certificate.py | 8 ++++---- sec_certs/dataset.py | 39 +++++++++++++++++---------------------- sec_certs/serialization.py | 3 ++- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/sec_certs/cert_processing.py b/sec_certs/cert_processing.py index bc11083b..0ccac952 100644 --- a/sec_certs/cert_processing.py +++ b/sec_certs/cert_processing.py @@ -1,9 +1,11 @@ from tqdm import tqdm from multiprocessing.pool import Pool, ThreadPool +from typing import Callable, Iterable, Optional import time -# TODO: Add timeout. Kinda meh with ThreadingTimeout, SignalTimeout does not work on Windows, stopit package. -def process_parallel(func, items, max_workers, callback=None, use_threading=True, progress_bar=True): + +def process_parallel(func: Callable, items: Iterable, max_workers: int, callback: Optional[Callable] = None, + use_threading: bool = True, progress_bar: bool = True): if use_threading is True: pool = ThreadPool(max_workers) else: diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index 724a60e0..a9ce04b7 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -382,13 +382,13 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): def to_dict(self): return copy.deepcopy(self.__dict__) - def __lt__(self, other): - return self.pp_name < other.pp_name - @classmethod def from_dict(cls, dct): return cls(*tuple(dct.values())) + def __lt__(self, other): + return self.pp_name < other.pp_name + def __init__(self, category: str, name: str, manufacturer: str, scheme: str, security_level: Union[str, set], not_valid_before: date, not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str], @@ -450,7 +450,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): self.src = self.src + ' + ' + other.src @classmethod - def from_dict(cls, dct: dict) -> 'CommonCriteriaCert': + def from_dict(cls, dct: Dict) -> 'CommonCriteriaCert': new_dct = dct.copy() new_dct['maintainance_updates'] = set(dct['maintainance_updates']) new_dct['protection_profiles'] = set(dct['protection_profiles']) diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 85b95b7c..277a84a9 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -3,7 +3,7 @@ import re from datetime import datetime import locale import logging -from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union +from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union, Generic, Optional, Sequence import json from importlib import import_module @@ -17,7 +17,7 @@ import requests from tabula import read_pdf import pandas as pd -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup, Tag from sec_certs.files import search_files @@ -116,7 +116,7 @@ class Dataset(ABC): logger.info(f'Failed to convert {path}, exit code: {e}') @staticmethod - def _download_parallel(urls, paths, prune_corrupted=True): + def _download_parallel(urls: Collection[str], paths: Collection[Path], prune_corrupted: bool = True): exit_codes = cert_processing.process_parallel(download.download_file, list(zip(urls, paths)), constants.N_THREADS) @@ -135,10 +135,6 @@ class Dataset(ABC): class CCDataset(Dataset, ComplexSerializableType): - def __init__(self, certs: Dict[str, 'CommonCriteriaCert'], root_dir: Path, name: str = 'dataset name', - description: str = 'dataset_description'): - super().__init__(certs, root_dir, name, description) - @property def web_dir(self) -> Path: return self.root_dir / 'web' @@ -223,7 +219,8 @@ class CCDataset(Dataset, ComplexSerializableType): logger.info( f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.') - def get_certs_from_web(self, to_download=True, keep_metadata: bool = True, get_active=True, get_archived=True): + def get_certs_from_web(self, to_download: bool = True, keep_metadata: bool = True, get_active: bool = True, + get_archived: bool = True): """ Downloads all metadata about certificates from CSV and HTML sources """ @@ -264,15 +261,13 @@ class CCDataset(Dataset, ComplexSerializableType): if not keep_metadata: shutil.rmtree(self.web_dir) - def _get_all_certs_from_csv(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: + def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> Dict[str, 'CommonCriteriaCert']: """ Creates dictionary of new certificates from csv sources. """ csv_sources = self.csv_products.keys() - csv_sources = [ - x for x in csv_sources if 'active' not in x or get_active] - csv_sources = [ - x for x in csv_sources if 'archived' not in x or get_archived] + csv_sources = [x for x in csv_sources if 'active' not in x or get_active] + csv_sources = [x for x in csv_sources if 'archived' not in x or get_archived] new_certs = {} for file in csv_sources: @@ -288,7 +283,7 @@ class CCDataset(Dataset, ComplexSerializableType): Using pandas, this parses a single CSV file. """ - def _get_primary_key_str(row): + def _get_primary_key_str(row: Tag): prim_key = row['category'] + row['cert_name'] + row['report_link'] return prim_key @@ -335,15 +330,13 @@ class CCDataset(Dataset, ComplexSerializableType): df_base.itertuples()} return certs - def _get_all_certs_from_html(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']: + def _get_all_certs_from_html(self, get_active: bool, get_archived: bool) -> Dict[str, 'CommonCriteriaCert']: """ Prepares dictionary of certificates from all html files. """ html_sources = self.html_products.keys() - html_sources = [ - x for x in html_sources if 'active' not in x or get_active] - html_sources = [ - x for x in html_sources if 'archived' not in x or get_archived] + html_sources = [x for x in html_sources if 'active' not in x or get_active] + html_sources = [x for x in html_sources if 'archived' not in x or get_archived] new_certs = {} for file in html_sources: @@ -422,12 +415,14 @@ class CCDataset(Dataset, ComplexSerializableType): def _download_reports(self): self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) reports_urls = [x.report_link for x in self] - self._download_parallel(reports_urls, self.report_pdf_paths.values(), prune_corrupted=True) + # for noqa below, see: https://youtrack.jetbrains.com/issue/PY-41771 + self._download_parallel(reports_urls, self.report_pdf_paths.values(), prune_corrupted=True) # noqa def _download_targets(self): self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) - target_urls = [x.st_link for x in self] - self._download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) + target_urls = [x.st_link for x in self.certs] + # for noqa below, see: https://youtrack.jetbrains.com/issue/PY-41771 + self._download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) # noqa def download_all_pdfs(self): logger.info('Downloading CC certificate reports') diff --git a/sec_certs/serialization.py b/sec_certs/serialization.py index d37dd067..c96fc07c 100644 --- a/sec_certs/serialization.py +++ b/sec_certs/serialization.py @@ -1,6 +1,7 @@ import json from datetime import date from pathlib import Path +from typing import Dict from abc import ABC, abstractmethod @@ -13,7 +14,7 @@ class ComplexSerializableType(ABC): @classmethod @abstractmethod - def from_dict(cls): + def from_dict(cls, dct: Dict): raise NotImplementedError -- cgit v1.3.1 From 3e8d73ad54c2e524b6ed19e3fe4c11158421bcce Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 27 Nov 2020 14:40:18 +0100 Subject: hotfix test --- .gitignore | 2 +- sec_certs/dataset.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 38b49e31..b1969bf4 100644 --- a/.gitignore +++ b/.gitignore @@ -111,4 +111,4 @@ venv.bak/ .mypy_cache/ # log -./cert_processing_log.txt \ No newline at end of file +./cc_processing_log.txt \ No newline at end of file diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 277a84a9..22ccfa6b 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -420,7 +420,7 @@ class CCDataset(Dataset, ComplexSerializableType): def _download_targets(self): self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) - target_urls = [x.st_link for x in self.certs] + target_urls = [x.st_link for x in self] # for noqa below, see: https://youtrack.jetbrains.com/issue/PY-41771 self._download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) # noqa -- cgit v1.3.1 From 7afc55f371b27894bae930145f4a359d98151d63 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 28 Nov 2020 12:17:22 +0100 Subject: Rethought way of processing certificates - Every certificate now implements a static method for its processing - These are called in parallel - There's some code duplication at the moment, stuff can be simplified. --- cc_oop_demo.py | 6 +- sec_certs/cert_processing.py | 4 +- sec_certs/certificate.py | 92 ++++++++++++++++++- sec_certs/dataset.py | 148 ++++++++++++++++++------------ sec_certs/helpers.py | 15 +++ test/data/test_cc_oop/fictional_cert.json | 9 +- test/data/test_cc_oop/toy_dataset.json | 18 +++- test/test_cc_oop.py | 60 ++++++------ 8 files changed, 252 insertions(+), 100 deletions(-) diff --git a/cc_oop_demo.py b/cc_oop_demo.py index c78759df..1da4cfe8 100644 --- a/cc_oop_demo.py +++ b/cc_oop_demo.py @@ -25,11 +25,11 @@ def main(): dset.get_certs_from_web() logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') - # Dump dataset into JSON + # # Dump dataset into JSON dset.to_json('./debug_dataset/cc_full_dataset.json') # Load dataset from JSON - new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json') + new_dset = CCDataset.from_json('/Users/adam/phd/projects/certificates/sec-certs/debug_dataset/cc_full_dataset.json') assert dset == new_dset @@ -37,7 +37,7 @@ def main(): dset.download_all_pdfs() # Convert pdfs to text - new_dset.convert_all_pdfs() + dset.convert_all_pdfs() end = datetime.now() logger.info(f'The computation took {(end-start)} seconds.') diff --git a/sec_certs/cert_processing.py b/sec_certs/cert_processing.py index 0ccac952..739bfa9f 100644 --- a/sec_certs/cert_processing.py +++ b/sec_certs/cert_processing.py @@ -11,7 +11,7 @@ def process_parallel(func: Callable, items: Iterable, max_workers: int, callback else: pool = Pool(max_workers) - results = [pool.apply_async(func, (*i, ), callback=callback) for i in items] + results = [pool.apply_async(func, (i, ), callback=callback) for i in items] if progress_bar is True: bar = tqdm(total=len(results)) @@ -25,4 +25,4 @@ def process_parallel(func: Callable, items: Iterable, max_workers: int, callback pool.close() pool.join() - return [r.get() for r in results] \ No newline at end of file + return [r.get() for r in results] diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index a9ce04b7..9f16ab42 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -6,6 +6,7 @@ from pathlib import Path import os import copy import json +import requests from abc import ABC, abstractmethod from bs4 import Tag, BeautifulSoup, NavigableString @@ -13,6 +14,7 @@ from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type from sec_certs import helpers, extract_certificates from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder +import sec_certs.constants as constants logger = logging.getLogger(__name__) @@ -249,7 +251,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): 'div', 'panel-body').children)[2].strip().split('\n')[1].strip() if html_items_found['fips_lab'] == '': - loggerr.warning(f"WARNING: NO LAB FOUND{current_file}") + logger.warning(f"WARNING: NO LAB FOUND{current_file}") if html_items_found['fips_nvlap_code'] == '': logger.warning(f"WARNING: NO NVLAP CODE FOUND{current_file}") @@ -389,12 +391,39 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): def __lt__(self, other): return self.pp_name < other.pp_name + @dataclass(init=False) + class InternalState(ComplexSerializableType): + st_link_ok: bool + report_link_ok: bool + st_convert_ok: bool + report_convert_ok: bool + st_pdf_path: Path + report_pdf_path: Path + st_txt_path: Path + report_txt_path: Path + + def __init__(self, st_link_ok: bool = True, report_link_ok: bool = True, + st_convert_ok: bool = True, report_convert_ok: bool = True): + self.st_link_ok = st_link_ok + self.report_link_ok = report_link_ok + self.st_convert_ok = st_convert_ok + self.report_convert_ok = report_convert_ok + + def to_dict(self): + return {'st_link_ok': self.st_link_ok, 'report_link_ok': self.report_link_ok, + 'st_convert_ok': self.st_convert_ok, 'report_convert_ok': self.report_convert_ok} + + @classmethod + def from_dict(cls, dct: Dict[str, bool]): + return cls(*tuple(dct.values())) + def __init__(self, category: str, name: str, manufacturer: str, scheme: str, security_level: Union[str, set], not_valid_before: date, not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str], manufacturer_web: Optional[str], protection_profiles: set, - maintainance_updates: set): + maintainance_updates: set, + state: Optional[InternalState]): super().__init__() self.category = category @@ -412,8 +441,10 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): self.protection_profiles = protection_profiles self.maintainance_updates = maintainance_updates - if self.st_link == self.empty_st_url: - self.st_link = None + if state is not None: + self.state = state + else: + self.state = self.InternalState() @property def dgst(self) -> str: @@ -442,6 +473,8 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): setattr(self, att, getattr(other, att)) elif att == 'src': pass # This is expected + elif att == 'state': + setattr(self, att, getattr(other, att)) else: if getattr(self, att) != getattr(other, att): logger.warning( @@ -566,4 +599,53 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): maintainance_div) if maintainance_div else set() return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, - st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances) + st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None) + + def set_local_paths(self, + report_pdf_dir: Optional[Union[str, Path]], + st_pdf_dir: Optional[Union[str, Path]], + report_txt_dir: Optional[Union[str, Path]], + st_txt_dir: Optional[Union[str, Path]]): + if report_pdf_dir is not None: + self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + '.pdf') + if st_pdf_dir is not None: + self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + '.pdf') + if report_txt_dir is not None: + self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + '.txt') + if st_txt_dir is not None: + self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + '.txt') + + @staticmethod + def download_pdf_report(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path) + if exit_code != requests.codes.ok: + logger.error(f'Failed to download report from {cert.report_link}, code: {exit_code}') + cert.state.report_link_ok = False + return cert + + @staticmethod + def download_pdf_target(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path) + if exit_code != requests.codes.ok: + logger.error(f'Cert dgst: {cert.dgst} failed to download report from {cert.report_link}, code: {exit_code}') + cert.state.st_link_ok = False + return cert + + def path_is_corrupted(self, local_path): + return local_path.exists() and local_path.stat().st_size >= constants.MIN_CORRECT_CERT_SIZE + + @staticmethod + def convert_report_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path, ['-raw']) + if exit_code != constants.RETURNCODE_OK: + logger.error(f'Cert dgst: {cert.dgst} failed to convert report pdf->txt') + cert.state.report_convert_ok = False + return cert + + @staticmethod + def convert_target_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path, ['-raw']) + if exit_code != constants.RETURNCODE_OK: + logger.error(f'Cert dgst: {cert.dgst} failed to convert security target pdf->txt') + cert.state.st_convert_ok = False + return cert diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 22ccfa6b..56466e2e 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -3,7 +3,7 @@ import re from datetime import datetime import locale import logging -from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union, Generic, Optional, Sequence +from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union, Generic, Optional, Sequence, Tuple import json from importlib import import_module @@ -37,13 +37,23 @@ logger = logging.getLogger(__name__) class Dataset(ABC): def __init__(self, certs: Dict[str, 'Certificate'], root_dir: Path, name: str = 'dataset name', description: str = 'dataset_description'): - self.root_dir = root_dir + self._root_dir = root_dir self.timestamp = datetime.now() self.sha256_digest = 'not implemented' self.name = name self.description = description self.certs = certs + @property + def root_dir(self): + return self._root_dir + + @root_dir.setter + def root_dir(self, new_dir: Union[str, Path]): + if not Path(new_dir).exists(): + raise FileNotFoundError('Root directory for Dataset does not exist') + self._root_dir = Path(new_dir) + def __iter__(self): for cert in self.certs.values(): yield cert @@ -81,9 +91,10 @@ class Dataset(ABC): @classmethod def from_json(cls, input_path: Union[str, Path]): - with Path(input_path).open('r') as handle: + input_path = Path(input_path) + with input_path.open('r') as handle: dset = json.load(handle, cls=CustomJSONDecoder) - dset.root_path = input_path.parent + dset.root_dir = input_path.parent.absolute() return dset @abstractmethod @@ -98,26 +109,9 @@ class Dataset(ABC): def download_all_pdfs(self): raise NotImplementedError('Not meant to be implemented by the base class.') - @staticmethod - def _convert_pdfs_to_txt(pdf_paths: Collection[Path], txt_paths: Collection[Path]): - assert len(pdf_paths) == len(txt_paths) - - partial_convert_pdf = partial(helpers.convert_pdf_file, options=['-raw']) - exit_codes = cert_processing.process_parallel(partial_convert_pdf, - list(zip(pdf_paths, txt_paths)), - constants.N_THREADS, - use_threading=False) - - n_successful = len([e for e in exit_codes if e == constants.RETURNCODE_OK]) - logger.info(f'Successfully converted {n_successful} files pdf->txt, {len(exit_codes) - n_successful} failed.') - - for path, e in zip(pdf_paths, exit_codes): - if e != constants.RETURNCODE_OK: - logger.info(f'Failed to convert {path}, exit code: {e}') - @staticmethod def _download_parallel(urls: Collection[str], paths: Collection[Path], prune_corrupted: bool = True): - exit_codes = cert_processing.process_parallel(download.download_file, + exit_codes = cert_processing.process_parallel(helpers.download_file, list(zip(urls, paths)), constants.N_THREADS) n_successful = len([e for e in exit_codes if e == requests.codes.ok]) @@ -135,6 +129,13 @@ class Dataset(ABC): class CCDataset(Dataset, ComplexSerializableType): + # TODO: Make properties propagate to changing internal state of related certificates + + @Dataset.root_dir.setter + def root_dir(self, new_dir: Union[str, Path]): + Dataset.root_dir.fset(self, new_dir) + self.set_local_paths() + @property def web_dir(self) -> Path: return self.root_dir / 'web' @@ -167,22 +168,6 @@ class CCDataset(Dataset, ComplexSerializableType): def targets_txt_dir(self) -> Path: return self.targets_dir / 'txt' - @property - def report_pdf_paths(self) -> Dict[str, Path]: - return {x: self.reports_pdf_dir / (self[x].dgst + '.pdf') for x in self.certs} - - @property - def report_txt_paths(self) -> Dict[str, Path]: - return {x: self.reports_txt_dir / (self[x].dgst + '.txt') for x in self.certs} - - @property - def target_pdf_paths(self) -> Dict[str, Path]: - return {x: self.targets_pdf_dir / (self[x].dgst + '.pdf') for x in self.certs} - - @property - def target_txt_paths(self) -> Dict[str, Path]: - return {x: self.targets_txt_dir / (self[x].dgst + '.txt') for x in self.certs} - html_products = { 'cc_products_active.html': 'https://www.commoncriteriaportal.org/products/', 'cc_products_archived.html': 'https://www.commoncriteriaportal.org/products/index.cfm?archived=1', @@ -202,6 +187,16 @@ class CCDataset(Dataset, ComplexSerializableType): 'cc_pp_archived.csv': 'https://www.commoncriteriaportal.org/pps/pps-archived.csv' } + @classmethod + def from_json(cls, input_path: Union[str, Path]): + dset = super().from_json(input_path) + dset.set_local_paths() + return dset + + def set_local_paths(self): + for cert in self: + cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir) + def _merge_certs(self, certs: Dict[str, 'CommonCriteriaCert']): """ Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates @@ -261,6 +256,8 @@ class CCDataset(Dataset, ComplexSerializableType): if not keep_metadata: shutil.rmtree(self.web_dir) + self.set_local_paths() + def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> Dict[str, 'CommonCriteriaCert']: """ Creates dictionary of new certificates from csv sources. @@ -326,7 +323,7 @@ class CCDataset(Dataset, ComplexSerializableType): certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level, x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv', - None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None)) for x in + None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None), None) for x in df_base.itertuples()} return certs @@ -412,41 +409,74 @@ class CCDataset(Dataset, ComplexSerializableType): return certs - def _download_reports(self): + def _download_reports(self, fresh=True): self.reports_pdf_dir.mkdir(parents=True, exist_ok=True) - reports_urls = [x.report_link for x in self] - # for noqa below, see: https://youtrack.jetbrains.com/issue/PY-41771 - self._download_parallel(reports_urls, self.report_pdf_paths.values(), prune_corrupted=True) # noqa - def _download_targets(self): + if fresh is True: + certs_to_process = self.certs.values() + else: + certs_to_process = [x for x in self.certs.values() if not x.state.report_link_ok] + + cert_processing.process_parallel(CommonCriteriaCert.download_pdf_report, certs_to_process, constants.N_THREADS) + + def _download_targets(self, fresh=True): self.targets_pdf_dir.mkdir(parents=True, exist_ok=True) - target_urls = [x.st_link for x in self] - # for noqa below, see: https://youtrack.jetbrains.com/issue/PY-41771 - self._download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) # noqa + if fresh is True: + certs_to_process = self.certs.values() + else: + certs_to_process = [x for x in self.certs.values() if not x.state.st_link_ok] + cert_processing.process_parallel(CommonCriteriaCert.download_pdf_target, certs_to_process, constants.N_THREADS) - def download_all_pdfs(self): + def download_all_pdfs(self, fresh: bool = True): logger.info('Downloading CC certificate reports') - self._download_reports() + self._download_reports(fresh) logger.info('Downloading CC security targets') - self._download_targets() + self._download_targets(fresh) + + if fresh is True: + # Attempt to re-download once + # TODO: Re-write the list comprehensions with filter? + if [x for x in self.certs.values() if not x.state.report_link_ok]: + logger.info('Attempting to re-download failed report links.') + self._download_reports(False) - def _convert_reports_to_txt(self): + if [x for x in self.certs.values() if not x.state.st_link_ok]: + logger.info('Attempting to re-download failed security target links.') + self._download_targets(False) + + def _convert_reports_to_txt(self, fresh: bool = True): self.reports_txt_dir.mkdir(parents=True, exist_ok=True) - # TODO: Get rid of the list() invocation here. - self._convert_pdfs_to_txt(list(self.report_pdf_paths.values()), list(self.report_txt_paths.values())) - def _convert_targets_to_txt(self): + if fresh is True: + certs_to_process = self.certs.values() + else: + certs_to_process = [x for x in self.certs.values() if not x.state.report_convert_ok] + cert_processing.process_parallel(CommonCriteriaCert.convert_report_pdf, certs_to_process, constants.N_THREADS) + + def _convert_targets_to_txt(self, fresh: bool = True): self.targets_txt_dir.mkdir(parents=True, exist_ok=True) - # TODO: Get rid of the list() invocation here. - self._convert_pdfs_to_txt(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values())) - def convert_all_pdfs(self): + if fresh is True: + certs_to_process = self.certs.values() + else: + certs_to_process = [x for x in self.certs.values() if not x.state.st_convert_ok] + cert_processing.process_parallel(CommonCriteriaCert.convert_target_pdf, certs_to_process, constants.N_THREADS) + + def convert_all_pdfs(self, fresh: bool = True): logger.info('Converting CC certificate reports to .txt') - self._convert_reports_to_txt() + self._convert_reports_to_txt(fresh) logger.info('Converting CC security targets to .txt') - self._convert_targets_to_txt() + self._convert_targets_to_txt(fresh) + + if fresh is True: + if [x for x in self.certs.values() if not x.state.report_convert_ok]: + logger.info('Attempting to re-convert failed report pdfs') + self._convert_reports_to_txt(False) + if [x for x in self.certs.values() if not x.state.st_convert_ok]: + logger.info('Attempting to re-convert failed target pdfs') + self._convert_targets_to_txt(False) class FIPSDataset(Dataset, ComplexSerializableType): diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 5219a897..80e490de 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -13,9 +13,24 @@ from datetime import date import numpy as np import pandas as pd import subprocess +import functools logger = logging.getLogger(__name__) +# Following two functions are from: https://stackoverflow.com/questions/31174295/getattr-and-setattr-on-nested-subobjects-chained-properties + + +def rsetattr(obj, attr, val): + pre, _, post = attr.rpartition('.') + return setattr(rgetattr(obj, pre) if pre else obj, post, val) + + +def rgetattr(obj, attr, *args): + def _getattr(obj, attr): + return getattr(obj, attr, *args) + return functools.reduce(_getattr, [obj] + attr.split('.')) + + def download_file(url: str, output: Path) -> int: try: r = requests.get(url, allow_redirects=True, timeout=5) diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json index 1aeaa0db..43e8e4d1 100644 --- a/test/data/test_cc_oop/fictional_cert.json +++ b/test/data/test_cc_oop/fictional_cert.json @@ -29,5 +29,12 @@ "maintainance_report_link": "https://maintainance.up", "maintainance_st_link": "https://maintainance.up" } - ] + ], + "state": { + "_type": "InternalState", + "st_link_ok": true, + "report_link_ok": true, + "st_convert_ok": true, + "report_convert_ok": true + } } \ No newline at end of file diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json index a16be012..ae98c9d1 100644 --- a/test/data/test_cc_oop/toy_dataset.json +++ b/test/data/test_cc_oop/toy_dataset.json @@ -24,7 +24,14 @@ "cert_link": "http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf", "manufacturer_web": "https://www.netiq.com/", "protection_profiles": [], - "maintainance_updates": [] + "maintainance_updates": [], + "state": { + "_type": "InternalState", + "st_link_ok": true, + "report_link_ok": true, + "st_convert_ok": true, + "report_convert_ok": true + } }, { "_type": "CommonCriteriaCert", @@ -47,7 +54,14 @@ "pp_link": "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf" } ], - "maintainance_updates": [] + "maintainance_updates": [], + "state": { + "_type": "InternalState", + "st_link_ok": true, + "report_link_ok": true, + "st_convert_ok": true, + "report_convert_ok": true + } } ] } \ No newline at end of file diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py index 98c11a4e..22c00929 100644 --- a/test/test_cc_oop.py +++ b/test/test_cc_oop.py @@ -8,7 +8,6 @@ import shutil import os from sec_certs.dataset import CCDataset -from sec_certs.serialization import CustomJSONEncoder, CustomJSONDecoder from sec_certs.certificate import CommonCriteriaCert import sec_certs.helpers as helpers @@ -17,20 +16,21 @@ class TestCommonCriteriaOOP(TestCase): def setUp(self): self.test_data_dir = Path(__file__).parent / 'data' / 'test_cc_oop' self.crt_one = CommonCriteriaCert('Access Control Devices and Systems', - 'NetIQ Identity Manager 4.7', - 'NetIQ Corporation', - 'SE', - {'ALC_FLR.2', - 'EAL3+'}, - date(2020, 6, 15), - date(2025, 6, 15), - 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf', - 'http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf', - 'csv + html', - 'http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf', - 'https://www.netiq.com/', - set(), - set()) + 'NetIQ Identity Manager 4.7', + 'NetIQ Corporation', + 'SE', + {'ALC_FLR.2', + 'EAL3+'}, + date(2020, 6, 15), + date(2025, 6, 15), + 'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf', + 'http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf', + 'csv + html', + 'http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf', + 'https://www.netiq.com/', + set(), + set(), + None) self.crt_two = CommonCriteriaCert('Access Control Devices and Systems', 'Magic SSO V4.0', @@ -46,7 +46,8 @@ class TestCommonCriteriaOOP(TestCase): 'https://www.dreamsecurity.com/', {CommonCriteriaCert.ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0', 'http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf')}, - set()) + set(), + None) pp = CommonCriteriaCert.ProtectionProfile('sample_pp', 'http://sample.pp') update = CommonCriteriaCert.MaintainanceReport(date(1900, 1, 1), 'Sample maintainance', 'https://maintainance.up', 'https://maintainance.up') @@ -63,7 +64,8 @@ class TestCommonCriteriaOOP(TestCase): 'http://path.to/cert/link', 'http://path.to/manufacturer/web', {pp}, - {update}) + {update}, + None) self.template_dataset = CCDataset({self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, Path('/fictional/path/to/dataset'), 'toy dataset', 'toy dataset description') self.template_dataset.timestamp = datetime(2020, 11, 16, hour=17, minute=4, second=14, microsecond=770153) @@ -81,27 +83,29 @@ class TestCommonCriteriaOOP(TestCase): 'Report link contains some improperly escaped characters.') def test_download_and_convert_pdfs(self): - with open(self.test_data_dir / 'toy_dataset.json', 'r') as handle: - dset = json.load(handle, cls=CustomJSONDecoder) + dset = CCDataset.from_json(self.test_data_dir / 'toy_dataset.json') with TemporaryDirectory() as td: dset.root_dir = Path(td) - dset.download_all_pdfs() dset.convert_all_pdfs() - actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.report_pdf_paths.items()} - actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.target_pdf_paths.items()} + actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.report_pdf_path) for key, val in dset.certs.items()} + actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.st_pdf_path) for key, val in dset.certs.items()} self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (certificate report) do not the template') self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template') - self.assertTrue(dset.report_txt_paths['869415cc4b91282e'].exists()) - self.assertTrue(dset.target_txt_paths['869415cc4b91282e'].exists()) - self.assertAlmostEqual(dset.target_txt_paths['869415cc4b91282e'].stat().st_size, - self.template_target_txt_path.stat().st_size, delta=1000) - self.assertAlmostEqual(dset.report_txt_paths['869415cc4b91282e'].stat().st_size, - self.template_report_txt_path.stat().st_size, delta=1000) + self.assertTrue(dset['869415cc4b91282e'].state.report_txt_path.exists()) + self.assertTrue(dset['869415cc4b91282e'].state.st_txt_path.exists()) + + self.assertAlmostEqual(dset['869415cc4b91282e'].state.st_txt_path.stat().st_size, + self.template_target_txt_path.stat().st_size, + delta=1000) + + self.assertAlmostEqual(dset['869415cc4b91282e'].state.report_txt_path.stat().st_size, + self.template_report_txt_path.stat().st_size, + delta=1000) def test_cert_to_json(self): with NamedTemporaryFile('w') as tmp: -- cgit v1.3.1 From ca86268aae6312faae024e7954b2faf938065b95 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 28 Nov 2020 12:32:31 +0100 Subject: fix hardcoded path, redundant functions --- cc_oop_demo.py | 2 +- sec_certs/helpers.py | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/cc_oop_demo.py b/cc_oop_demo.py index 1da4cfe8..a49a2e7d 100644 --- a/cc_oop_demo.py +++ b/cc_oop_demo.py @@ -29,7 +29,7 @@ def main(): dset.to_json('./debug_dataset/cc_full_dataset.json') # Load dataset from JSON - new_dset = CCDataset.from_json('/Users/adam/phd/projects/certificates/sec-certs/debug_dataset/cc_full_dataset.json') + new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json') assert dset == new_dset diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 80e490de..afea2476 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -17,19 +17,6 @@ import functools logger = logging.getLogger(__name__) -# Following two functions are from: https://stackoverflow.com/questions/31174295/getattr-and-setattr-on-nested-subobjects-chained-properties - - -def rsetattr(obj, attr, val): - pre, _, post = attr.rpartition('.') - return setattr(rgetattr(obj, pre) if pre else obj, post, val) - - -def rgetattr(obj, attr, *args): - def _getattr(obj, attr): - return getattr(obj, attr, *args) - return functools.reduce(_getattr, [obj] + attr.split('.')) - def download_file(url: str, output: Path) -> int: try: -- cgit v1.3.1 From 0b4e066fa07248377118e4d548e6a9257db50ab6 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 28 Nov 2020 18:44:54 +0100 Subject: Implement improvements proposed by J08nY in review --- sec_certs/certificate.py | 2 +- sec_certs/constants.py | 1 + sec_certs/dataset.py | 56 +++++++++++++++++++++--------------------------- sec_certs/helpers.py | 5 +++-- 4 files changed, 30 insertions(+), 34 deletions(-) diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py index 9f16ab42..04c53baa 100644 --- a/sec_certs/certificate.py +++ b/sec_certs/certificate.py @@ -632,7 +632,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType): return cert def path_is_corrupted(self, local_path): - return local_path.exists() and local_path.stat().st_size >= constants.MIN_CORRECT_CERT_SIZE + return not local_path.exists() or local_path.stat().st_size < constants.MIN_CORRECT_CERT_SIZE @staticmethod def convert_report_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': diff --git a/sec_certs/constants.py b/sec_certs/constants.py index c3bf646e..6d4aade5 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -3,6 +3,7 @@ from enum import Enum N_THREADS = 8 RESPONSE_OK = 200 RETURNCODE_OK = 0 +REQUEST_TIMEOUT = 5 MIN_CORRECT_CERT_SIZE = 5000 diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py index 56466e2e..e261b98f 100644 --- a/sec_certs/dataset.py +++ b/sec_certs/dataset.py @@ -3,33 +3,25 @@ import re from datetime import datetime import locale import logging -from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union, Generic, Optional, Sequence, Tuple +from typing import Dict, List, ClassVar, Collection, Union import json from importlib import import_module - -import copy from abc import ABC, abstractmethod from pathlib import Path import shutil - -from functools import partial import requests - from tabula import read_pdf import pandas as pd from bs4 import BeautifulSoup, Tag - -from sec_certs.files import search_files -from sec_certs import helpers as helpers -from sec_certs.helpers import find_tables, repair_pdf -from sec_certs.certificate import CommonCriteriaCert, Certificate, FIPSCertificate -from sec_certs.extract_certificates import extract_certificates_keywords -from sec_certs.constants import FIPS_NOT_AVAILABLE_CERT_SIZE +import sec_certs.helpers as helpers import sec_certs.constants as constants -import sec_certs.download as download import sec_certs.cert_processing as cert_processing +import sec_certs.files as files + +from sec_certs.certificate import CommonCriteriaCert, Certificate, FIPSCertificate from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder +from sec_certs.extract_certificates import extract_certificates_keywords logger = logging.getLogger(__name__) @@ -50,13 +42,12 @@ class Dataset(ABC): @root_dir.setter def root_dir(self, new_dir: Union[str, Path]): - if not Path(new_dir).exists(): + if not (new_path := Path(new_dir)).exists(): raise FileNotFoundError('Root directory for Dataset does not exist') - self._root_dir = Path(new_dir) + self._root_dir = new_path def __iter__(self): - for cert in self.certs.values(): - yield cert + yield from self.certs.values() def __getitem__(self, item: str) -> 'Certificate': return self.certs.__getitem__(item.lower()) @@ -82,7 +73,8 @@ class Dataset(ABC): def from_dict(cls, dct: Dict): certs = {x.dgst: x for x in dct['certs']} dset = cls(certs, Path('./'), dct['name'], dct['description']) - assert len(dset) == dct['n_certs'] + if len(dset) != (claimed := dct['n_certs']): + logger.error(f'The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed}).') return dset def to_json(self, output_path: Union[str, Path]): @@ -332,8 +324,10 @@ class CCDataset(Dataset, ComplexSerializableType): Prepares dictionary of certificates from all html files. """ html_sources = self.html_products.keys() - html_sources = [x for x in html_sources if 'active' not in x or get_active] - html_sources = [x for x in html_sources if 'archived' not in x or get_archived] + if get_active is False: + html_sources = filter(lambda x: 'active' not in x, html_sources) + if get_archived is False: + html_sources = filter(lambda x: 'archived' not in x, html_sources) new_certs = {} for file in html_sources: @@ -435,13 +429,12 @@ class CCDataset(Dataset, ComplexSerializableType): self._download_targets(fresh) if fresh is True: - # Attempt to re-download once - # TODO: Re-write the list comprehensions with filter? - if [x for x in self.certs.values() if not x.state.report_link_ok]: + # Attempt to re-download once if some files are missing + if any(filter(lambda x: not x.state.report_link_ok, self.certs.values())): logger.info('Attempting to re-download failed report links.') self._download_reports(False) - if [x for x in self.certs.values() if not x.state.st_link_ok]: + if any(filter(lambda x: not x.state.st_link_ok, self.certs.values())): logger.info('Attempting to re-download failed security target links.') self._download_targets(False) @@ -471,10 +464,11 @@ class CCDataset(Dataset, ComplexSerializableType): self._convert_targets_to_txt(fresh) if fresh is True: - if [x for x in self.certs.values() if not x.state.report_convert_ok]: + # Attempt to re-convert once if some files failed + if any(filter(lambda x: not x.state.report_convert_ok, self.certs.values())): logger.info('Attempting to re-convert failed report pdfs') self._convert_reports_to_txt(False) - if [x for x in self.certs.values() if not x.state.st_convert_ok]: + if any(filter(lambda x: not x.state.st_convert_ok, self.certs.values())): logger.info('Attempting to re-convert failed target pdfs') self._convert_targets_to_txt(False) @@ -512,7 +506,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): for i in self.certs: if not (self.policies_dir / f'{i}.pdf').exists(): missing.append(i) - elif os.path.getsize(self.policies_dir / f'{i}.pdf') < FIPS_NOT_AVAILABLE_CERT_SIZE: + elif os.path.getsize(self.policies_dir / f'{i}.pdf') < constants.FIPS_NOT_AVAILABLE_CERT_SIZE: not_available.append(i) return missing, not_available @@ -597,7 +591,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): :return: list of files that couldn't have been decoded """ - list_of_files = search_files(self.policies_dir) + list_of_files = files.search_files(self.policies_dir) not_decoded = [] for cert_file in list_of_files: cert_file = Path(cert_file) @@ -611,7 +605,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): continue with open(cert_file, 'r') as f: - tables = find_tables(f.read(), cert_file) + tables = helpers.find_tables(f.read(), cert_file) # If we find any tables with page numbers, we process them if tables: @@ -621,7 +615,7 @@ class FIPSDataset(Dataset, ComplexSerializableType): pages=tables, silent=True) except Exception: try: - repair_pdf(cert_file.with_suffix('')) + helpers.repair_pdf(cert_file.with_suffix('')) data = read_pdf(cert_file.with_suffix( ''), pages=tables, silent=True) diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index afea2476..67b90347 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -13,14 +13,15 @@ from datetime import date import numpy as np import pandas as pd import subprocess -import functools +import sec_certs.constants as constants + logger = logging.getLogger(__name__) def download_file(url: str, output: Path) -> int: try: - r = requests.get(url, allow_redirects=True, timeout=5) + r = requests.get(url, allow_redirects=True, timeout=constants.REQUEST_TIMEOUT) if r.status_code == requests.codes.ok: with output.open("wb") as f: f.write(r.content) -- cgit v1.3.1