From 47e548aaa8a13cdd124be0a039d5e43df3e1fbcb Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 22 Oct 2021 15:26:29 +0200 Subject: move config and add better error message to invalid jsonschema --- examples/cc_cpe_labeling.py | 2 +- examples/cc_oop_demo.py | 2 +- examples/fips_oop_demo.py | 2 +- sec_certs/certificate/common_criteria.py | 3 +- sec_certs/certificate/fips.py | 4 +- sec_certs/config/configuration.py | 35 ++++++ sec_certs/config/settings-schema.json | 178 +++++++++++++++++++++++++++++++ sec_certs/config/settings.yaml | 40 +++++++ sec_certs/configuration.py | 36 ------- sec_certs/dataset/common_criteria.py | 7 +- sec_certs/dataset/cve.py | 2 +- sec_certs/dataset/dataset.py | 2 +- sec_certs/dataset/fips.py | 6 +- sec_certs/dataset/fips_algorithm.py | 3 +- sec_certs/settings-schema.json | 178 ------------------------------- sec_certs/settings.yaml | 40 ------- tests/test_fips_oop.py | 2 +- 17 files changed, 268 insertions(+), 274 deletions(-) create mode 100644 sec_certs/config/configuration.py create mode 100644 sec_certs/config/settings-schema.json create mode 100644 sec_certs/config/settings.yaml delete mode 100644 sec_certs/configuration.py delete mode 100644 sec_certs/settings-schema.json delete mode 100644 sec_certs/settings.yaml diff --git a/examples/cc_cpe_labeling.py b/examples/cc_cpe_labeling.py index e467cfa9..d0d0548d 100644 --- a/examples/cc_cpe_labeling.py +++ b/examples/cc_cpe_labeling.py @@ -3,7 +3,7 @@ import logging from pathlib import Path from sec_certs.dataset.common_criteria import CCDataset -from sec_certs.configuration import config +from sec_certs.config.configuration import config logger = logging.getLogger(__name__) diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py index c89832fe..1f2919e8 100644 --- a/examples/cc_oop_demo.py +++ b/examples/cc_oop_demo.py @@ -3,7 +3,7 @@ from datetime import datetime import logging from sec_certs.dataset.common_criteria import CCDataset -from sec_certs.configuration import config +from sec_certs.config.configuration import config logger = logging.getLogger(__name__) diff --git a/examples/fips_oop_demo.py b/examples/fips_oop_demo.py index dd74e866..0ae3a444 100644 --- a/examples/fips_oop_demo.py +++ b/examples/fips_oop_demo.py @@ -4,7 +4,7 @@ import logging import click from sec_certs.dataset.fips import FIPSDataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.configuration import config +from sec_certs.config.configuration import config @click.command() diff --git a/sec_certs/certificate/common_criteria.py b/sec_certs/certificate/common_criteria.py index b64eadd8..0446b1b0 100644 --- a/sec_certs/certificate/common_criteria.py +++ b/sec_certs/certificate/common_criteria.py @@ -1,7 +1,6 @@ import copy import itertools import operator -import re from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path @@ -16,7 +15,7 @@ from sec_certs.dataset.cpe import CPE, CPEDataset from sec_certs.dataset.cve import CVEDataset from sec_certs.serialization import ComplexSerializableType from sec_certs.certificate.protection_profile import ProtectionProfile -from sec_certs.configuration import config +from sec_certs.config.configuration import config class CommonCriteriaCert(Certificate, ComplexSerializableType): diff --git a/sec_certs/certificate/fips.py b/sec_certs/certificate/fips.py index d90fe835..6d131ab5 100644 --- a/sec_certs/certificate/fips.py +++ b/sec_certs/certificate/fips.py @@ -11,10 +11,10 @@ from dateutil import parser from tabula import read_pdf import sec_certs.constants -from sec_certs import helpers, dataset, constants as constants +from sec_certs import helpers, constants as constants from sec_certs.cert_rules import fips_common_rules, REGEXEC_SEP, fips_rules from sec_certs.certificate.certificate import Certificate, logger -from sec_certs.configuration import config +from sec_certs.config.configuration import config from sec_certs.constants import LINE_SEPARATOR from sec_certs.helpers import save_modified_cert_file, normalize_match_string, load_cert_file from sec_certs.serialization import ComplexSerializableType diff --git a/sec_certs/config/configuration.py b/sec_certs/config/configuration.py new file mode 100644 index 00000000..7a3b8831 --- /dev/null +++ b/sec_certs/config/configuration.py @@ -0,0 +1,35 @@ +import yaml +from typing import Union +from pathlib import Path +import jsonschema +import json + + +class Configuration(object): + def load(self, filepath: Union[str, Path]): + with Path(filepath).open('r') as file: + state = yaml.load(file, Loader=yaml.FullLoader) + + script_dir = Path(__file__).parent + + with (Path(script_dir) / 'settings-schema.json').open('r') as file: + schema = json.loads(file.read()) + + try: + jsonschema.validate(state, schema) + except jsonschema.exceptions.ValidationError as e: + print(f'{e}\n\nIn file {filepath}') + + for k, v in state.items(): + setattr(self, k, v) + + def __getattribute__(self, key): + res = object.__getattribute__(self, key) + if isinstance(res, dict) and 'value' in res: + return res['value'] + return object.__getattribute__(self, key) + + +config_path = Path(__file__).parent / 'settings.yaml' +config = Configuration() +config.load(config_path) diff --git a/sec_certs/config/settings-schema.json b/sec_certs/config/settings-schema.json new file mode 100644 index 00000000..e05d06e5 --- /dev/null +++ b/sec_certs/config/settings-schema.json @@ -0,0 +1,178 @@ +{ + "title": "settings for sec-certs", + "type": "object", + "definitions": { + "settings_string_entry": { + "required": [ + "description", + "value" + ], + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "settings_boolean_entry": { + "type": "object", + "required": [ + "description", + "value" + ], + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "boolean" + } + } + }, + "settings_number_entry": { + "type": "object", + "required": [ + "description", + "value" + ], + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "number" + } + } + }, + "settings_url_entry": { + "type": "object", + "required": [ + "description", + "value" + ], + "properties": { + "description": { + "type": "string" + }, + "value": { + "type": "string", + "format": "uri", + "pattern": "^(https?|http?)://", + "minLength": 1, + "maxLength": 255 + } + } + } + }, + "properties": { + "log_filepath": { + "$ref": "#/definitions/settings_string_entry" + }, + "smallest_certificate_id_to_connect": { + "$ref": "#/definitions/settings_number_entry" + }, + "year_difference_between_validations": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 0 + } + } + } + ] + }, + "use_text_with_newlines_during_parsing": { + "$ref": "#/definitions/settings_boolean_entry" + }, + "n_threads": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 1 + } + } + } + ] + }, + "cc_cpe_matching_threshold": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 0, + "maximum": 100 + } + } + } + ] + }, + "cc_cpe_max_matches": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "exclusiveMinimum": 0 + } + } + } + ] + }, + "cc_latest_snapshot": { + "$ref": "#/definitions/settings_url_entry" + }, + "cc_maintenances_latest_snapshot": { + "$ref": "#/definitions/settings_url_entry" + }, + "ignore_first_page": { + "$ref": "#/definitions/settings_boolean_entry" + }, + "cert_threshold": { + "allOf": [ + { + "$ref": "#/definitions/settings_number_entry" + }, + { + "properties": { + "value": { + "minimum": 0 + } + } + } + ] + }, + "fips_latest_snapshot": { + "$ref": "#/definitions/settings_url_entry" + } + }, + "required": [ + "log_filepath", + "smallest_certificate_id_to_connect", + "year_difference_between_validations", + "use_text_with_newlines_during_parsing", + "n_threads", + "cc_cpe_matching_threshold", + "cc_cpe_max_matches", + "cc_latest_snapshot", + "cc_maintenances_latest_snapshot", + "ignore_first_page", + "cert_threshold", + "fips_latest_snapshot" + ] +} \ No newline at end of file diff --git a/sec_certs/config/settings.yaml b/sec_certs/config/settings.yaml new file mode 100644 index 00000000..a5c8911f --- /dev/null +++ b/sec_certs/config/settings.yaml @@ -0,0 +1,40 @@ +--- +log_filepath: + description: Path to the file, relative to working directory, where the log will be stored + value: ./cert_processing_log.txt +smallest_certificate_id_to_connect: + description: During validation we don't connect certificates with number lower than + _this_ to connections + value: 40 +year_difference_between_validations: + description: During validation we don't connect certificates with validation dates + difference higher than _this_ + value: 7 +use_text_with_newlines_during_parsing: + description: During keyword search, search in text with newlines + value: true +n_threads: + description: How many threads to use for parallel computations + value: 8 +cc_cpe_matching_threshold: + description: Level of required string similarity between CPE and certificate name on CC CPE matching, 0-100. Lower values yield more false negatives, higher values more false positives + value: 70 +cc_cpe_max_matches: + description: Maximum number of candidate CPE items that may be related to given certificate, >0 + value: 20 +cc_latest_snapshot: + description: Url from where to fetch the latest snapshot of fully processed CC dataset + value: https://www.ajanovsky.cz/cc_latest_snapshot.json +cc_maintenances_latest_snapshot: + description: Url from where to fetch the latest snapshot of CC maintenance updates + value: https://www.ajanovsky.cz/cc_maintenances_latest_snapshot.json +ignore_first_page: + description: During keyword search, first page usually contains addresses - ignore it. + value: true +cert_threshold: + description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty. + value: 5 +fips_latest_snapshot: + description: Url for the latest snapshot of FIPS dataset + value: https://seccerts.org/static/fips_dset_ad8734a39b856ca1a1b7b073713872359bae7545.json + diff --git a/sec_certs/configuration.py b/sec_certs/configuration.py deleted file mode 100644 index 6a495e60..00000000 --- a/sec_certs/configuration.py +++ /dev/null @@ -1,36 +0,0 @@ -import yaml -from typing import Union, Any -from pathlib import Path -from importlib_resources import files -import jsonschema -import json -import os - -import sec_certs - - -class Configuration(object): - def load(self, filepath: Union[str, Path]): - with Path(filepath).open('r') as file: - state = yaml.load(file, Loader=yaml.FullLoader) - - script_dir = Path(__file__).parent - - with open(Path(script_dir) / 'settings-schema.json', 'r') as file: - schema = json.loads(file.read()) - - jsonschema.validate(state, schema) - - for k, v in state.items(): - setattr(self, k, v) - - def __getattribute__(self, key): - res = object.__getattribute__(self, key) - if isinstance(res, dict) and 'value' in res: - return res['value'] - return object.__getattribute__(self, key) - - -config_path = files(sec_certs).joinpath('settings.yaml') -config = Configuration() -config.load(config_path) diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py index 430b7b33..f50fdf67 100644 --- a/sec_certs/dataset/common_criteria.py +++ b/sec_certs/dataset/common_criteria.py @@ -7,7 +7,7 @@ import time from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Dict, Optional, Union, List, Tuple, Set +from typing import Dict, Optional, Union, List, Tuple import json import numpy as np @@ -15,15 +15,14 @@ import pandas as pd from bs4 import Tag, BeautifulSoup from tqdm import tqdm -from sec_certs import helpers as helpers, parallel_processing as cert_processing, constants as constants -from sec_certs.dataset.cpe import CPEDataset, CPE +from sec_certs import helpers as helpers, parallel_processing as cert_processing from sec_certs.dataset.cve import CVEDataset from sec_certs.dataset.dataset import Dataset, logger from sec_certs.serialization import ComplexSerializableType, serialize, CustomJSONDecoder from sec_certs.certificate.common_criteria import CommonCriteriaCert from sec_certs.dataset.protection_profile import ProtectionProfileDataset from sec_certs.certificate.protection_profile import ProtectionProfile -from sec_certs.configuration import config +from sec_certs.config.configuration import config from sec_certs.certificate.cc_maintenance_update import CommonCriteriaMaintenanceUpdate diff --git a/sec_certs/dataset/cve.py b/sec_certs/dataset/cve.py index a69b648f..e8d70a55 100644 --- a/sec_certs/dataset/cve.py +++ b/sec_certs/dataset/cve.py @@ -15,7 +15,7 @@ from sec_certs.parallel_processing import process_parallel import sec_certs.constants as constants import sec_certs.helpers as helpers from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder -from sec_certs.configuration import config +from sec_certs.config.configuration import config logger = logging.getLogger(__name__) diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py index bb4469b6..0842122d 100644 --- a/sec_certs/dataset/dataset.py +++ b/sec_certs/dataset/dataset.py @@ -14,7 +14,7 @@ import sec_certs.parallel_processing as cert_processing from sec_certs.certificate.certificate import Certificate from sec_certs.serialization import CustomJSONDecoder, CustomJSONEncoder -from sec_certs.configuration import config +from sec_certs.config.configuration import config from sec_certs.serialization import serialize from sec_certs.dataset.cpe import CPEDataset diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py index 354aec8d..9f39af5c 100644 --- a/sec_certs/dataset/fips.py +++ b/sec_certs/dataset/fips.py @@ -3,19 +3,17 @@ import logging import os from itertools import groupby from pathlib import Path -from typing import ClassVar, Tuple, List, Dict, Optional, Union -import json +from typing import Tuple, List, Dict, Optional, Union from bs4 import BeautifulSoup from graphviz import Digraph from sec_certs import constants as constants, parallel_processing as cert_processing, helpers as helpers -from sec_certs.configuration import config +from sec_certs.config.configuration import config from sec_certs.dataset.dataset import Dataset, logger from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset from sec_certs.serialization import ComplexSerializableType, serialize from sec_certs.certificate.fips import FIPSCertificate -from sec_certs.dataset.cpe import CPEDataset class FIPSDataset(Dataset, ComplexSerializableType): diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py index 42f1b22e..2ef3f22d 100644 --- a/sec_certs/dataset/fips_algorithm.py +++ b/sec_certs/dataset/fips_algorithm.py @@ -5,12 +5,11 @@ from typing import Dict, Union, List from bs4 import BeautifulSoup -import sec_certs.helpers from sec_certs import helpers as helpers, constants as constants, parallel_processing as cert_processing from sec_certs.dataset.dataset import Dataset from sec_certs.serialization import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder from sec_certs.certificate.fips import FIPSCertificate -from sec_certs.configuration import config +from sec_certs.config.configuration import config class FIPSAlgorithmDataset(Dataset, ComplexSerializableType): diff --git a/sec_certs/settings-schema.json b/sec_certs/settings-schema.json deleted file mode 100644 index e05d06e5..00000000 --- a/sec_certs/settings-schema.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "title": "settings for sec-certs", - "type": "object", - "definitions": { - "settings_string_entry": { - "required": [ - "description", - "value" - ], - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "string" - } - } - }, - "settings_boolean_entry": { - "type": "object", - "required": [ - "description", - "value" - ], - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "boolean" - } - } - }, - "settings_number_entry": { - "type": "object", - "required": [ - "description", - "value" - ], - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "number" - } - } - }, - "settings_url_entry": { - "type": "object", - "required": [ - "description", - "value" - ], - "properties": { - "description": { - "type": "string" - }, - "value": { - "type": "string", - "format": "uri", - "pattern": "^(https?|http?)://", - "minLength": 1, - "maxLength": 255 - } - } - } - }, - "properties": { - "log_filepath": { - "$ref": "#/definitions/settings_string_entry" - }, - "smallest_certificate_id_to_connect": { - "$ref": "#/definitions/settings_number_entry" - }, - "year_difference_between_validations": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 0 - } - } - } - ] - }, - "use_text_with_newlines_during_parsing": { - "$ref": "#/definitions/settings_boolean_entry" - }, - "n_threads": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 1 - } - } - } - ] - }, - "cc_cpe_matching_threshold": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 0, - "maximum": 100 - } - } - } - ] - }, - "cc_cpe_max_matches": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "exclusiveMinimum": 0 - } - } - } - ] - }, - "cc_latest_snapshot": { - "$ref": "#/definitions/settings_url_entry" - }, - "cc_maintenances_latest_snapshot": { - "$ref": "#/definitions/settings_url_entry" - }, - "ignore_first_page": { - "$ref": "#/definitions/settings_boolean_entry" - }, - "cert_threshold": { - "allOf": [ - { - "$ref": "#/definitions/settings_number_entry" - }, - { - "properties": { - "value": { - "minimum": 0 - } - } - } - ] - }, - "fips_latest_snapshot": { - "$ref": "#/definitions/settings_url_entry" - } - }, - "required": [ - "log_filepath", - "smallest_certificate_id_to_connect", - "year_difference_between_validations", - "use_text_with_newlines_during_parsing", - "n_threads", - "cc_cpe_matching_threshold", - "cc_cpe_max_matches", - "cc_latest_snapshot", - "cc_maintenances_latest_snapshot", - "ignore_first_page", - "cert_threshold", - "fips_latest_snapshot" - ] -} \ No newline at end of file diff --git a/sec_certs/settings.yaml b/sec_certs/settings.yaml deleted file mode 100644 index a5c8911f..00000000 --- a/sec_certs/settings.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -log_filepath: - description: Path to the file, relative to working directory, where the log will be stored - value: ./cert_processing_log.txt -smallest_certificate_id_to_connect: - description: During validation we don't connect certificates with number lower than - _this_ to connections - value: 40 -year_difference_between_validations: - description: During validation we don't connect certificates with validation dates - difference higher than _this_ - value: 7 -use_text_with_newlines_during_parsing: - description: During keyword search, search in text with newlines - value: true -n_threads: - description: How many threads to use for parallel computations - value: 8 -cc_cpe_matching_threshold: - description: Level of required string similarity between CPE and certificate name on CC CPE matching, 0-100. Lower values yield more false negatives, higher values more false positives - value: 70 -cc_cpe_max_matches: - description: Maximum number of candidate CPE items that may be related to given certificate, >0 - value: 20 -cc_latest_snapshot: - description: Url from where to fetch the latest snapshot of fully processed CC dataset - value: https://www.ajanovsky.cz/cc_latest_snapshot.json -cc_maintenances_latest_snapshot: - description: Url from where to fetch the latest snapshot of CC maintenance updates - value: https://www.ajanovsky.cz/cc_maintenances_latest_snapshot.json -ignore_first_page: - description: During keyword search, first page usually contains addresses - ignore it. - value: true -cert_threshold: - description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty. - value: 5 -fips_latest_snapshot: - description: Url for the latest snapshot of FIPS dataset - value: https://seccerts.org/static/fips_dset_ad8734a39b856ca1a1b7b073713872359bae7545.json - diff --git a/tests/test_fips_oop.py b/tests/test_fips_oop.py index a212d5b9..e7f3a01c 100644 --- a/tests/test_fips_oop.py +++ b/tests/test_fips_oop.py @@ -4,7 +4,7 @@ from tempfile import TemporaryDirectory from sec_certs.dataset.fips import FIPSDataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.configuration import config +from sec_certs.config.configuration import config from tests.fips_test_utils import generate_html -- cgit v1.3.1