merge dev into cve-matching

17 files changed, 232 insertions, 29 deletions

diff --git a/examples/cc_cpe_labeling.py b/examples/cc_cpe_labeling.py
index e467cfa9..d0d0548d 100644
--- a/examples/cc_cpe_labeling.py
+++ b/examples/cc_cpe_labeling.py
@@ -3,7 +3,7 @@ import logging
 from pathlib import Path
 
 from sec_certs.dataset.common_criteria import CCDataset
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 
 logger = logging.getLogger(__name__)
 
diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py
index c89832fe..1f2919e8 100644
--- a/examples/cc_oop_demo.py
+++ b/examples/cc_oop_demo.py
@@ -3,7 +3,7 @@ from datetime import datetime
 import logging
 
 from sec_certs.dataset.common_criteria import CCDataset
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 
 logger = logging.getLogger(__name__)
 
diff --git a/examples/fips_oop_demo.py b/examples/fips_oop_demo.py
index dd74e866..0ae3a444 100644
--- a/examples/fips_oop_demo.py
+++ b/examples/fips_oop_demo.py
@@ -4,7 +4,7 @@ import logging
 import click
 from sec_certs.dataset.fips import FIPSDataset
 from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 
 
 @click.command()
diff --git a/requirements.txt b/requirements.txt
index a663632c..79f24599 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,3 +29,4 @@ rapidfuzz==1.1.1
 pyyaml
 importlib-resources==5.1.2
 scikit-learn
+jsonschema==3.2.0
\ No newline at end of file
diff --git a/sec_certs/configuration.py b/sec_certs/config/configuration.py
index 9ec7c2be..7a3b8831 100644
--- a/sec_certs/configuration.py
+++ b/sec_certs/config/configuration.py
@@ -1,17 +1,25 @@
 import yaml
-from typing import Union, Any
+from typing import Union
 from pathlib import Path
-from importlib_resources import files
-
-import sec_certs
+import jsonschema
+import json
 
 
 class Configuration(object):
-    # TODO: Should raise ValueError on unvalidated config
     def load(self, filepath: Union[str, Path]):
         with Path(filepath).open('r') as file:
             state = yaml.load(file, Loader=yaml.FullLoader)
 
+        script_dir = Path(__file__).parent
+
+        with (Path(script_dir) / 'settings-schema.json').open('r') as file:
+            schema = json.loads(file.read())
+
+        try:
+            jsonschema.validate(state, schema)
+        except jsonschema.exceptions.ValidationError as e:
+            print(f'{e}\n\nIn file {filepath}')
+
         for k, v in state.items():
             setattr(self, k, v)
 
@@ -22,6 +30,6 @@ class Configuration(object):
         return object.__getattribute__(self, key)
 
 
-config_path = files(sec_certs).joinpath('settings.yaml')
+config_path = Path(__file__).parent / 'settings.yaml'
 config = Configuration()
 config.load(config_path)
diff --git a/sec_certs/config/settings-schema.json b/sec_certs/config/settings-schema.json
new file mode 100644
index 00000000..e05d06e5
--- /dev/null
+++ b/sec_certs/config/settings-schema.json
@@ -0,0 +1,178 @@
+{
+    "title": "settings for sec-certs",
+    "type": "object",
+    "definitions": {
+        "settings_string_entry": {
+            "required": [
+                "description",
+                "value"
+            ],
+            "type": "object",
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "value": {
+                    "type": "string"
+                }
+            }
+        },
+        "settings_boolean_entry": {
+            "type": "object",
+            "required": [
+                "description",
+                "value"
+            ],
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "value": {
+                    "type": "boolean"
+                }
+            }
+        },
+        "settings_number_entry": {
+            "type": "object",
+            "required": [
+                "description",
+                "value"
+            ],
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "value": {
+                    "type": "number"
+                }
+            }
+        },
+        "settings_url_entry": {
+            "type": "object",
+            "required": [
+                "description",
+                "value"
+            ],
+            "properties": {
+                "description": {
+                    "type": "string"
+                },
+                "value": {
+                    "type": "string",
+                    "format": "uri",
+                    "pattern": "^(https?|http?)://",
+                    "minLength": 1,
+                    "maxLength": 255
+                }
+            }
+        }
+    },
+    "properties": {
+        "log_filepath": {
+            "$ref": "#/definitions/settings_string_entry"
+        },
+        "smallest_certificate_id_to_connect": {
+            "$ref": "#/definitions/settings_number_entry"
+        },
+        "year_difference_between_validations": {
+            "allOf": [
+                {
+                    "$ref": "#/definitions/settings_number_entry"
+                },
+                {
+                    "properties": {
+                        "value": {
+                            "minimum": 0
+                        }
+                    }
+                }
+            ]
+        },
+        "use_text_with_newlines_during_parsing": {
+            "$ref": "#/definitions/settings_boolean_entry"
+        },
+        "n_threads": {
+            "allOf": [
+                {
+                    "$ref": "#/definitions/settings_number_entry"
+                },
+                {
+                    "properties": {
+                        "value": {
+                            "minimum": 1
+                        }
+                    }
+                }
+            ]
+        },
+        "cc_cpe_matching_threshold": {
+            "allOf": [
+                {
+                    "$ref": "#/definitions/settings_number_entry"
+                },
+                {
+                    "properties": {
+                        "value": {
+                            "minimum": 0,
+                            "maximum": 100
+                        }
+                    }
+                }
+            ]
+        },
+        "cc_cpe_max_matches": {
+            "allOf": [
+                {
+                    "$ref": "#/definitions/settings_number_entry"
+                },
+                {
+                    "properties": {
+                        "value": {
+                            "exclusiveMinimum": 0
+                        }
+                    }
+                }
+            ]
+        },
+        "cc_latest_snapshot": {
+            "$ref": "#/definitions/settings_url_entry"
+        },
+        "cc_maintenances_latest_snapshot": {
+            "$ref": "#/definitions/settings_url_entry"
+        },
+        "ignore_first_page": {
+            "$ref": "#/definitions/settings_boolean_entry"
+        },
+        "cert_threshold": {
+            "allOf": [
+                {
+                    "$ref": "#/definitions/settings_number_entry"
+                },
+                {
+                    "properties": {
+                        "value": {
+                            "minimum": 0
+                        }
+                    }
+                }
+            ]
+        },
+        "fips_latest_snapshot": {
+            "$ref": "#/definitions/settings_url_entry"
+        }
+    },
+    "required": [
+        "log_filepath",
+        "smallest_certificate_id_to_connect",
+        "year_difference_between_validations",
+        "use_text_with_newlines_during_parsing",
+        "n_threads",
+        "cc_cpe_matching_threshold",
+        "cc_cpe_max_matches",
+        "cc_latest_snapshot",
+        "cc_maintenances_latest_snapshot",
+        "ignore_first_page",
+        "cert_threshold",
+        "fips_latest_snapshot"
+    ]
+}
\ No newline at end of file
diff --git a/sec_certs/settings.yaml b/sec_certs/config/settings.yaml
index 9788f4bf..9788f4bf 100644
--- a/sec_certs/settings.yaml
+++ b/sec_certs/config/settings.yaml
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index d5a43008..5b11df4e 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -7,7 +7,7 @@ import time
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, Optional, Union, List, Tuple, Set
+from typing import Dict, Optional, Union, List, Tuple
 import json
 
 import numpy as np
@@ -15,17 +15,13 @@ import pandas as pd
 from bs4 import Tag, BeautifulSoup
 from tqdm import tqdm
 
-from sec_certs import helpers as helpers, parallel_processing as cert_processing, constants as constants
-from sec_certs.dataset.cpe import CPEDataset
-from sec_certs.sample.cpe import CPE
+from sec_certs import helpers as helpers, parallel_processing as cert_processing
 from sec_certs.dataset.cve import CVEDataset
-from sec_certs.sample.cve import CVE
 from sec_certs.dataset.dataset import Dataset, logger
 from sec_certs.serialization import ComplexSerializableType, serialize, CustomJSONDecoder
 from sec_certs.sample.common_criteria import CommonCriteriaCert
 from sec_certs.dataset.protection_profile import ProtectionProfileDataset
 from sec_certs.sample.protection_profile import ProtectionProfile
-from sec_certs.configuration import config
 from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate
 
 
diff --git a/sec_certs/dataset/cve.py b/sec_certs/dataset/cve.py
index 2b1555de..ca518dac 100644
--- a/sec_certs/dataset/cve.py
+++ b/sec_certs/dataset/cve.py
@@ -18,7 +18,7 @@ import sec_certs.helpers as helpers
 from sec_certs.sample.cve import CVE
 from sec_certs.sample.cpe import CPE
 from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 
 logger = logging.getLogger(__name__)
 
diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py
index 49cb2c24..c29ceef8 100644
--- a/sec_certs/dataset/dataset.py
+++ b/sec_certs/dataset/dataset.py
@@ -14,7 +14,7 @@ import sec_certs.parallel_processing as cert_processing
 
 from sec_certs.sample.certificate import Certificate
 from sec_certs.serialization import CustomJSONDecoder, CustomJSONEncoder
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 from sec_certs.serialization import serialize
 from sec_certs.dataset.cpe import CPEDataset
 
diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py
index c769f27e..e9af75b2 100644
--- a/sec_certs/dataset/fips.py
+++ b/sec_certs/dataset/fips.py
@@ -3,19 +3,17 @@ import logging
 import os
 from itertools import groupby
 from pathlib import Path
-from typing import ClassVar, Tuple, List, Dict, Optional, Union
-import json
+from typing import Tuple, List, Dict, Optional, Union
 
 from bs4 import BeautifulSoup
 from graphviz import Digraph
 
 from sec_certs import constants as constants, parallel_processing as cert_processing, helpers as helpers
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 from sec_certs.dataset.dataset import Dataset, logger
 from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset
 from sec_certs.serialization import ComplexSerializableType, serialize
 from sec_certs.sample.fips import FIPSCertificate
-from sec_certs.dataset.cpe import CPEDataset
 
 
 class FIPSDataset(Dataset, ComplexSerializableType):
diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py
index b957033b..71e7d571 100644
--- a/sec_certs/dataset/fips_algorithm.py
+++ b/sec_certs/dataset/fips_algorithm.py
@@ -5,12 +5,12 @@ from typing import Dict, Union, List
 
 from bs4 import BeautifulSoup
 
-import sec_certs.helpers
 from sec_certs import helpers as helpers, constants as constants, parallel_processing as cert_processing
 from sec_certs.dataset.dataset import Dataset
 from sec_certs.serialization import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder
+
 from sec_certs.sample.fips import FIPSCertificate
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 
 
 class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py
index 744a2f82..838ccca3 100644
--- a/sec_certs/sample/common_criteria.py
+++ b/sec_certs/sample/common_criteria.py
@@ -1,7 +1,6 @@
 import copy
 import itertools
 import operator
-import re
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path
@@ -20,7 +19,8 @@ from sec_certs.dataset.cve import CVEDataset
 from sec_certs.sample.cve import CVE
 from sec_certs.serialization import ComplexSerializableType
 from sec_certs.sample.protection_profile import ProtectionProfile
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
+
 
 
 class CommonCriteriaCert(Certificate, ComplexSerializableType):
diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py
index 4373fc5b..309fca8c 100644
--- a/sec_certs/sample/fips.py
+++ b/sec_certs/sample/fips.py
@@ -11,10 +11,11 @@ from dateutil import parser
 from tabula import read_pdf
 
 import sec_certs.constants
-from sec_certs import helpers, dataset, constants as constants
+from sec_certs import helpers, constants as constants
 from sec_certs.cert_rules import fips_common_rules, REGEXEC_SEP, fips_rules
+
 from sec_certs.sample.certificate import Certificate, logger
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 from sec_certs.constants import LINE_SEPARATOR
 from sec_certs.helpers import save_modified_cert_file, normalize_match_string, load_cert_file
 from sec_certs.serialization import ComplexSerializableType
diff --git a/setup.py b/setup.py
index c1e97aa0..7b28c85e 100644
--- a/setup.py
+++ b/setup.py
@@ -34,7 +34,7 @@ setup(
     },
     include_package_data=True,
     package_data={
-        'sec_certs': ['settings.yaml']
+        'sec_certs': ['settings.yaml', 'settings-schema.json']
     },
     entry_points={
         'console_scripts': [
diff --git a/tests/settings_test.yaml b/tests/settings_test.yaml
index 9b07a8be..a5c8911f 100644
--- a/tests/settings_test.yaml
+++ b/tests/settings_test.yaml
@@ -1,4 +1,7 @@
 ---
+log_filepath:
+  description: Path to the file, relative to working directory, where the log will be stored
+  value: ./cert_processing_log.txt
 smallest_certificate_id_to_connect:
   description: During validation we don't connect certificates with number lower than
     _this_ to connections
@@ -10,10 +13,28 @@ year_difference_between_validations:
 use_text_with_newlines_during_parsing:
   description: During keyword search, search in text with newlines
   value: true
+n_threads:
+  description: How many threads to use for parallel computations
+  value: 8
+cc_cpe_matching_threshold:
+  description: Level of required string similarity between CPE and certificate name on CC CPE matching, 0-100. Lower values yield more false negatives, higher values more false positives
+  value: 70
+cc_cpe_max_matches:
+  description: Maximum number of candidate CPE items that may be related to given certificate, >0
+  value: 20
+cc_latest_snapshot:
+  description: Url from where to fetch the latest snapshot of fully processed CC dataset
+  value: https://www.ajanovsky.cz/cc_latest_snapshot.json
+cc_maintenances_latest_snapshot:
+  description: Url from where to fetch the latest snapshot of CC maintenance updates
+  value: https://www.ajanovsky.cz/cc_maintenances_latest_snapshot.json
 ignore_first_page:
   description: During keyword search, first page usually contains addresses - ignore it.
   value: true
 cert_threshold:
   description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty.
   value: 5
+fips_latest_snapshot:
+  description: Url for the latest snapshot of FIPS dataset
+  value: https://seccerts.org/static/fips_dset_ad8734a39b856ca1a1b7b073713872359bae7545.json
 
diff --git a/tests/test_fips_oop.py b/tests/test_fips_oop.py
index a212d5b9..e7f3a01c 100644
--- a/tests/test_fips_oop.py
+++ b/tests/test_fips_oop.py
@@ -4,7 +4,7 @@ from tempfile import TemporaryDirectory
 
 from sec_certs.dataset.fips import FIPSDataset
 from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset
-from sec_certs.configuration import config
+from sec_certs.config.configuration import config
 from tests.fips_test_utils import generate_html