Merge branch 'dev' into feature/tqdm-configuration

22 files changed, 343 insertions, 211 deletions

diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py
index b7c87835..f1e7184f 100644
--- a/sec_certs/cert_rules.py
+++ b/sec_certs/cert_rules.py
@@ -1,5 +1,6 @@
 import copy
 import re
+from typing import Dict, List, Pattern, Union
 
 REGEXEC_SEP = r'[ ,;\]”)(]'
 
@@ -508,13 +509,16 @@ rules.update(common_rules)
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #                            For FIPS
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-fips_rules = {}
-fips_rules['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids
-fips_rules['rules_to_remove'] = rules_fips_to_remove
-fips_rules['rules_security_level'] = rules_fips_security_level
-fips_rules['rules_cert_id'] = rules_fips_cert
+fips_rules_base: Dict[str, List[str]] = {}
+fips_rules_base['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids
+fips_rules_base['rules_to_remove'] = rules_fips_to_remove
+fips_rules_base['rules_security_level'] = rules_fips_security_level
+fips_rules_base['rules_cert_id'] = rules_fips_cert
 fips_common_rules = copy.deepcopy(common_rules)  # make separate copy not to process cc rules by fips's re.compile
 
-for rule in fips_rules:
-    for current_rule in range(len(fips_rules[rule])):
-        fips_rules[rule][current_rule] = re.compile(fips_rules[rule][current_rule])
+fips_rules: Dict[str, List[Pattern[str]]] = {}
+
+for rule in fips_rules_base:
+    fips_rules[rule] = []
+    for current_rule in range(len(fips_rules_base[rule])):
+        fips_rules[rule].append(re.compile(fips_rules_base[rule][current_rule]))
diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index 9c261f23..86f06933 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -1,6 +1,7 @@
 from enum import Enum
 
 RESPONSE_OK = 200
+# TODO: types don't match
 RETURNCODE_OK = 'ok'
 RETURNCODE_NOK = 'nok'
 REQUEST_TIMEOUT = 10
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index 81394c5e..130943a1 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -6,7 +6,7 @@ import tempfile
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Dict, Optional, Union, List, Tuple, Set, ClassVar
+from typing import Dict, Iterator, Optional, Set, Union, List, Tuple, Mapping, ClassVar
 import json
 
 import numpy as np
@@ -17,6 +17,7 @@ from sec_certs import helpers as helpers, parallel_processing as cert_processing
 from sec_certs.dataset.dataset import Dataset, logger
 from sec_certs.serialization.json import ComplexSerializableType, serialize, CustomJSONDecoder
 from sec_certs.sample.common_criteria import CommonCriteriaCert
+from sec_certs.sample.certificate import Certificate
 from sec_certs.dataset.protection_profile import ProtectionProfileDataset
 from sec_certs.sample.protection_profile import ProtectionProfile
 from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate
@@ -36,8 +37,9 @@ class CCDataset(Dataset, ComplexSerializableType):
             return any(vars(self))
 
     certs: Dict[str, 'CommonCriteriaCert']
+    # TODO: Figure out how to type this. The problem is that this breaks covariance of the types, which mypy doesn't allow.
 
-    def __init__(self, certs: Dict[str, 'CommonCriteriaCert'], root_dir: Path, name: str = 'dataset name',
+    def __init__(self, certs: Mapping[str, 'Certificate'], root_dir: Path, name: str = 'dataset name',
                  description: str = 'dataset_description', state: Optional[DatasetInternalState] = None):
         super().__init__(certs, root_dir, name, description)
 
@@ -45,7 +47,7 @@ class CCDataset(Dataset, ComplexSerializableType):
             state = self.DatasetInternalState()
         self.state = state
 
-    def __iter__(self) -> CommonCriteriaCert:
+    def __iter__(self) -> Iterator[CommonCriteriaCert]:
         yield from self.certs.values()
 
     def to_dict(self):
@@ -68,10 +70,10 @@ class CCDataset(Dataset, ComplexSerializableType):
         dset.state = copy.deepcopy(dct['state'])
         return dset
 
-    @Dataset.root_dir.setter
+    @Dataset.root_dir.setter # type: ignore
     def root_dir(self, new_dir: Union[str, Path]):
         old_dset = copy.deepcopy(self)
-        Dataset.root_dir.fset(self, new_dir)
+        Dataset.root_dir.fset(self, new_dir) # type: ignore
         self.set_local_paths()
 
         if self.state and old_dset.root_dir != Path('..'):
@@ -216,6 +218,8 @@ class CCDataset(Dataset, ComplexSerializableType):
         pp_dataset = constructor[to_download](self.pp_dataset_path)
 
         for cert in self:
+            if cert.protection_profiles is None:
+                raise RuntimeError("Building of the dataset probably failed - this should not be happening.")
             cert.protection_profiles = {pp_dataset.pps.get((x.pp_name, x.pp_link), x) for x in cert.protection_profiles}
 
         if not keep_metadata:
@@ -251,7 +255,7 @@ class CCDataset(Dataset, ComplexSerializableType):
         """
         Creates dictionary of new certificates from csv sources.
         """
-        csv_sources = self.CSV_PRODUCTS_URL.keys()
+        csv_sources = list(self.CSV_PRODUCTS_URL.keys())
         csv_sources = [x for x in csv_sources if 'active' not in x or get_active]
         csv_sources = [x for x in csv_sources if 'archived' not in x or get_archived]
 
@@ -322,7 +326,7 @@ class CCDataset(Dataset, ComplexSerializableType):
         profiles = {x.dgst: set([ProtectionProfile(y) for y in
                                  helpers.sanitize_protection_profiles(x.protection_profiles)]) for x in
                     df_base.itertuples()}
-        updates = {x.dgst: set() for x in df_base.itertuples()}
+        updates: Dict[str, Set] = {x.dgst: set() for x in df_base.itertuples()}
         for x in df_main.itertuples():
             updates[x.dgst].add(CommonCriteriaCert.MaintenanceReport(x.maintenance_date.date(), x.maintenance_title,
                                                                       x.maintenance_report_link,
@@ -341,11 +345,11 @@ class CCDataset(Dataset, ComplexSerializableType):
         """
         Prepares dictionary of certificates from all html files.
         """
-        html_sources = self.HTML_PRODUCTS_URL.keys()
+        html_sources = list(self.HTML_PRODUCTS_URL.keys())
         if get_active is False:
-            html_sources = filter(lambda x: 'active' not in x, html_sources)
+            html_sources = [x for x in html_sources if 'active' not in x]
         if get_archived is False:
-            html_sources = filter(lambda x: 'archived' not in x, html_sources)
+            html_sources = [x for x in html_sources if 'archived' not in x]
 
         new_certs = {}
         for file in html_sources:
@@ -633,7 +637,7 @@ class CCDataset(Dataset, ComplexSerializableType):
 
         self.state.certs_analyzed = True
 
-    def get_certs_from_name(self, cert_name: str) -> List[CommonCriteriaCert]:
+    def get_certs_from_name(self, cert_name: str) -> List[Certificate]:
         return [crt for crt in self if crt.name == cert_name]
 
     def process_maintenance_updates(self):
@@ -660,8 +664,11 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType):
     """
     Should be used merely for actions related to Maintenance updates: download pdfs, convert pdfs, extract data from pdfs
     """
-
-    def __init__(self, certs: Dict[str, 'CommonCriteriaMaintenanceUpdate'], root_dir: Path, name: str = 'dataset name',
+    # TODO: Types - if I use dictionary in CCDataset, I can't use more specific dictionary here (otherwise the CCDataset
+    # one would have to be a Mapping - not mutable)
+    certs: Dict[str, 'CommonCriteriaMaintenanceUpdate'] # type: ignore
+    def __init__(self, certs: Mapping[str, 'Certificate'],
+                 root_dir: Path, name: str = 'dataset name',
                  description: str = 'dataset_description', state: Optional[CCDataset.DatasetInternalState] = None):
         super().__init__(certs, root_dir, name, description, state)
         self.state.meta_sources_parsed = True
@@ -670,7 +677,7 @@ class CCDatasetMaintenanceUpdates(CCDataset, ComplexSerializableType):
     def certs_dir(self) -> Path:
         return self.root_dir
 
-    def __iter__(self) -> CommonCriteriaMaintenanceUpdate:
+    def __iter__(self) -> Iterator[CommonCriteriaMaintenanceUpdate]:
         yield from self.certs.values()
 
     def _compute_heuristics(self, download_fresh_cpes: bool = False):
diff --git a/sec_certs/dataset/cpe.py b/sec_certs/dataset/cpe.py
index 38fe7908..faf82b05 100644
--- a/sec_certs/dataset/cpe.py
+++ b/sec_certs/dataset/cpe.py
@@ -89,8 +89,16 @@ class CPEDataset(ComplexSerializableType):
         root = ET.parse(xml_path).getroot()
         dct = {}
         for cpe_item in root.findall('{http://cpe.mitre.org/dictionary/2.0}cpe-item'):
-            title = cpe_item.find('{http://cpe.mitre.org/dictionary/2.0}title').text
-            cpe_uri = cpe_item.find('{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item').attrib['name']
+            found_title = cpe_item.find('{http://cpe.mitre.org/dictionary/2.0}title')
+            if found_title is None:
+                raise RuntimeError("Title is not found during building CPE dataset from xml - this should not be happening")
+            title = found_title.text
+            
+            found_cpe_uri = cpe_item.find('{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item')
+            if found_cpe_uri is None:
+                raise RuntimeError("CPE uri is not found during building CPE dataset from xml - this should not be happening")
+            cpe_uri = found_cpe_uri.attrib['name']
+            
             dct[cpe_uri] = CPE(cpe_uri, title)
         return cls(False, Path(json_path), dct)
 
@@ -114,6 +122,8 @@ class CPEDataset(ComplexSerializableType):
         if isinstance(cve_dset, (str, Path)):
             cve_dset = CVEDataset.from_json(cve_dset)
 
+        if not isinstance(cve_dset, CVEDataset):
+            raise RuntimeError("Conversion of CVE dataset did not work.")
         all_cpes_in_cve_dset = set(itertools.chain.from_iterable([cve.vulnerable_cpes for cve in cve_dset]))
 
         old_len = len(self.cpes)
diff --git a/sec_certs/dataset/cve.py b/sec_certs/dataset/cve.py
index ef83b5b5..5a77386b 100644
--- a/sec_certs/dataset/cve.py
+++ b/sec_certs/dataset/cve.py
@@ -1,6 +1,6 @@
 import itertools
 from dataclasses import dataclass, field
-from typing import Dict, List,  Optional, Union, Final, Set
+from typing import Dict, List,  Optional, Tuple, Union, Final, Set
 import datetime
 from pathlib import Path
 import tempfile
@@ -46,7 +46,7 @@ class CVEDataset(ComplexSerializableType):
     def __len__(self) -> int:
         return len(self.cves)
 
-    def __eq__(self, other: 'CVEDataset'):
+    def __eq__(self, other: object):
         return isinstance(other, CVEDataset) and self.cves == other.cves
 
     def build_lookup_dict(self, use_nist_mapping: bool = True, nist_matching_filepath: Optional[Path] = None):
@@ -79,8 +79,8 @@ class CVEDataset(ComplexSerializableType):
                     self.cpe_to_cve_ids_lookup[cpe.uri].append(cve.cve_id)
 
     @classmethod
-    def download_cves(cls, output_path: str, start_year: int, end_year: int):
-        output_path = Path(output_path)
+    def download_cves(cls, output_path_str: str, start_year: int, end_year: int):
+        output_path = Path(output_path_str)
         if not output_path.exists:
             output_path.mkdir()
 
@@ -123,7 +123,9 @@ class CVEDataset(ComplexSerializableType):
 
         return cls(all_cves)
 
-    def to_json(self, output_path: str):
+    def to_json(self, output_path: Optional[Union[str, Path]] = None):
+        if output_path is None:
+            raise RuntimeError(f"You tried to serialize an object ({type(self)}) that does not have implicit json path. Please provide json_path.")
         with Path(output_path).open('w') as handle:
             json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
 
@@ -162,7 +164,7 @@ class CVEDataset(ComplexSerializableType):
         df = pd.DataFrame([x.pandas_tuple for x in self], columns=CVE.pandas_columns)
         return df.set_index('cve_id')
 
-    def get_nist_cpe_matching_dict(self, input_filepath: Optional[Union[str, Path]]):
+    def get_nist_cpe_matching_dict(self, input_filepath: Optional[Path]):
         def parse_key_cpe(field: Dict) -> CPE:
             start_version = None
             if 'versionStartIncluding' in field:
@@ -180,7 +182,7 @@ class CVEDataset(ComplexSerializableType):
 
         def parse_values_cpe(field: Dict) -> List[CPE]:
             return [CPE(x['cpe23Uri']) for x in field['cpe_name']]
-
+        
         logger.debug('Attempting to get NIST mapping file.')
         if not input_filepath or not input_filepath.is_file():
             logger.debug('NIST mapping file not available, going to download.')
diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py
index 15e59e01..be06416c 100644
--- a/sec_certs/dataset/dataset.py
+++ b/sec_certs/dataset/dataset.py
@@ -1,6 +1,6 @@
 from datetime import datetime
 import logging
-from typing import Dict, Collection, Union, List, Tuple
+from typing import Dict, Collection, Optional, Set, Union, List, Tuple, Mapping
 
 import json
 from abc import ABC, abstractmethod
@@ -25,8 +25,8 @@ from sec_certs.model.cpe_matching import CPEClassifier
 logger = logging.getLogger(__name__)
 
 
-class Dataset(ABC, ComplexSerializableType):
-    def __init__(self, certs: Dict[str, 'Certificate'], root_dir: Path, name: str = 'dataset name',
+class Dataset(ABC):
+    def __init__(self, certs: Mapping[str, 'Certificate'], root_dir: Path, name: str = 'dataset name',
                  description: str = 'dataset_description'):
         self._root_dir = root_dir
         self.timestamp = datetime.now()
@@ -81,12 +81,14 @@ class Dataset(ABC, ComplexSerializableType):
         return self.certs.__getitem__(item.lower())
 
     def __setitem__(self, key: str, value: 'Certificate'):
-        self.certs.__setitem__(key.lower(), value)
+        self.certs.__setitem__(key.lower(), value) # type: ignore
 
     def __len__(self) -> int:
         return len(self.certs)
 
-    def __eq__(self, other: 'Dataset') -> bool:
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Dataset):
+            return NotImplemented
         return self.certs == other.certs
 
     def __str__(self) -> str:
@@ -125,7 +127,7 @@ class Dataset(ABC, ComplexSerializableType):
         raise NotImplementedError('Not meant to be implemented by the base class.')
 
     @abstractmethod
-    def download_all_pdfs(self):
+    def download_all_pdfs(self, cert_ids: Optional[Set[str]] = None):
         raise NotImplementedError('Not meant to be implemented by the base class.')
 
     @staticmethod
@@ -186,7 +188,7 @@ class Dataset(ABC, ComplexSerializableType):
             """
             if cpe.title and (cpe.version == '-' or cpe.version == '*') and not any(char.isdigit() for char in cpe.title):
                 return False
-            elif not cpe.title and (cpe.version == '-' or cpe.version == '*') and not any(char.isdigit() for char in cpe.item_name):
+            elif not cpe.title and cpe.item_name and (cpe.version == '-' or cpe.version == '*') and not any(char.isdigit() for char in cpe.item_name):
                 return False
             return True
 
@@ -236,7 +238,7 @@ class Dataset(ABC, ComplexSerializableType):
             match_keys = [x.lstrip('$') for x in match_keys]
             predicted_annotations = [annotation[x] for x in match_keys if annotation[x] != 'No good match']
 
-            cpes = set()
+            cpes: Set[Optional[CPE]] = set()
             for x in predicted_annotations:
                 if x not in cpe_dset.title_to_cpes:
                     print(f'Error: {x} not in dataset')
@@ -245,7 +247,9 @@ class Dataset(ABC, ComplexSerializableType):
                     if to_update and not cpes:
                         cpes = to_update
                     elif to_update and cpes:
-                        cpes = cpes.update(to_update)
+                    # TODO: This was here like cpes = cpes.update(to_update), but update() does not return anything.
+                    # Did you try to hack something using that or was that just a typo?
+                        cpes.update(to_update)
 
 
             # cpes = set(itertools.chain.from_iterable([cpe_dset.title_to_cpes.get(x, []) for x in predicted_annotations]))
@@ -259,7 +263,7 @@ class Dataset(ABC, ComplexSerializableType):
             certs = self.get_certs_from_name(cert_name)
 
             for c in certs:
-                c.heuristics.verified_cpe_matches = {x.uri for x in cpes} if cpes else None
+                c.heuristics.verified_cpe_matches = {x.uri for x in cpes if x is not None} if cpes else None
 
     def get_certs_from_name(self, name: str) -> List[Certificate]:
         raise NotImplementedError('Not meant to be implemented by the base class.')
diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py
index a1886e07..f252bffc 100644
--- a/sec_certs/dataset/fips.py
+++ b/sec_certs/dataset/fips.py
@@ -1,15 +1,17 @@
+import datetime
 import tempfile
 import logging
 import os
 from itertools import groupby
 from pathlib import Path
-from typing import Tuple, List, Dict, Optional
 
-from bs4 import BeautifulSoup
+from typing import Set, Tuple, List, Dict, Optional, Mapping
+from bs4 import BeautifulSoup, NavigableString
 from graphviz import Digraph
 
 from sec_certs import constants as constants, parallel_processing as cert_processing, helpers as helpers
 from sec_certs.config.configuration import config
+from sec_certs.sample.certificate import Certificate
 from sec_certs.dataset.dataset import Dataset, logger
 from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset
 from sec_certs.serialization.json import ComplexSerializableType, serialize
@@ -23,11 +25,11 @@ class FIPSDataset(Dataset, ComplexSerializableType):
     certs: Dict[str, FIPSCertificate]
 
     def __init__(
-        self, certs: dict, root_dir: Path, name: str = "dataset name", description: str = "dataset_description"
+        self, certs: Mapping[str, 'Certificate'], root_dir: Path, name: str = "dataset name", description: str = "dataset_description"
     ):
         super().__init__(certs, root_dir, name, description)
-        self.keywords = {}
-        self.algorithms = None
+        self.keywords: Dict[str, Dict] = {}
+        self.algorithms: Optional[FIPSAlgorithmDataset] = None
         self.new_files = 0
 
     @property
@@ -49,13 +51,13 @@ class FIPSDataset(Dataset, ComplexSerializableType):
     # After web scan, there should be a FIPSCertificate object created for every entry
     @property
     def successful_web_scan(self) -> bool:
-        return all(self.certs) and all(cert.web_scan for cert in self.certs.values())
+        return all(self.certs) and all(cert.web_scan for cert in self.certs.values() if cert is not None)
 
     @property
     def successful_pdf_scan(self) -> bool:
-        return all(cert.pdf_scan for cert in self.certs.values())
+        return all(cert.pdf_scan for cert in self.certs.values() if cert is not None)
 
-    def get_certs_from_name(self, module_name: str) -> List[FIPSCertificate]:
+    def get_certs_from_name(self, module_name: str) -> List[Certificate]:
         return [crt for crt in self if crt.web_scan.module_name == module_name]
 
     def find_empty_pdfs(self) -> Tuple[List, List]:
@@ -86,7 +88,6 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
     def match_algs(self) -> Dict:
         output = {}
-        cert: FIPSCertificate
         for cert in self.certs.values():
             # if the pdf has not been processed, no matching can be done
             if not cert.pdf_scan.keywords or not cert.state.txt_state:
@@ -98,13 +99,13 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         output = {k: v for k, v in output.items() if v != 0}
         return output
 
-    def download_all_pdfs(self):
+    def download_all_pdfs(self, cert_ids: Optional[Set[str]] = None):
         sp_paths, sp_urls = [], []
         self.policies_dir.mkdir(exist_ok=True)
-
-        for cert_id in list(self.certs.keys()):
-            if not (self.policies_dir / f"{cert_id}.pdf").exists() or (
-                self.certs[cert_id] and not self.certs[cert_id].state.txt_state
+        if cert_ids is None:
+            raise RuntimeError("You need to provide cert ids to FIPS download PDFs functionality.")
+        for cert_id in cert_ids:
+            if not (self.policies_dir / f"{cert_id}.pdf").exists()  or (cert_id in self.certs and not self.certs[cert_id].state.txt_state
             ):
                 sp_urls.append(
                     f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf"
@@ -116,11 +117,11 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         )
         self.new_files += len(sp_urls)
 
-    def download_all_htmls(self) -> List[str]:
+    def download_all_htmls(self, cert_ids: Set[str]) -> List[str]:
         html_paths, html_urls = [], []
         new_files = []
         self.web_dir.mkdir(exist_ok=True)
-        for cert_id in self.certs.keys():
+        for cert_id in cert_ids:
             if not (self.web_dir / f"{cert_id}.html").exists():
                 html_urls.append(
                     f"https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{cert_id}"
@@ -135,8 +136,9 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         failed = [c for c in failed if c]
 
         self.new_files += len(html_urls)
-        logger.info(f"Download failed for {len(failed)} files. Retrying...")
-        cert_processing.process_parallel(FIPSCertificate.download_html_page, failed, config.n_threads, progress_bar_desc="Downloading HTML files again")
+        if len(failed) != 0:
+            logger.info(f"Download failed for {len(failed)} files. Retrying...")
+            cert_processing.process_parallel(FIPSCertificate.download_html_page, failed, config.n_threads, progress_bar_desc="Downloading HTML files again")
         return new_files
 
     @serialize
@@ -149,49 +151,56 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         ]
         cert_processing.process_parallel(FIPSCertificate.convert_pdf_file, tuples, config.n_threads, progress_bar_desc="Converting to txt")
 
-    def prepare_dataset(self, test: Optional[Path] = None, update: bool = False):
+    def prepare_dataset(self, test: Optional[Path] = None, update: bool = False) -> Set[str]:
         if test:
             html_files = [test]
         else:
-            html_files = ["fips_modules_active.html", "fips_modules_historical.html", "fips_modules_revoked.html"]
+            html_files = [Path("fips_modules_active.html"), Path("fips_modules_historical.html"), Path("fips_modules_revoked.html")]
             helpers.download_file(
                 "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0",
-                self.web_dir / "fips_modules_active.html",
+                Path(self.web_dir / "fips_modules_active.html"),
             )
             helpers.download_file(
                 "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Historical&ValidationYear=0",
-                self.web_dir / "fips_modules_historical.html",
+                Path(self.web_dir / "fips_modules_historical.html"),
             )
             helpers.download_file(
                 "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Revoked&ValidationYear=0",
-                self.web_dir / "fips_modules_revoked.html",
+                Path(self.web_dir / "fips_modules_revoked.html"),
             )
 
         # Parse those files and get list of currently processable files (always)
+        cert_ids: Set[str] = set()
         for f in html_files:
-            self._get_certificates_from_html(self.web_dir / f, update)
+            cert_ids |= self._get_certificates_from_html(self.web_dir / f, update)
+            
+        return cert_ids
 
-    def download_neccessary_files(self):
-        self.download_all_htmls()
-        # self.download_all_pdfs()
+    def download_neccessary_files(self, cert_ids: Set[str]):
+        self.download_all_htmls(cert_ids)
+        self.download_all_pdfs(cert_ids)
 
-    def _get_certificates_from_html(self, html_file: Path, update: bool = False) -> None:
-        logger.info(f"Getting sample ids from {html_file}")
+    def _get_certificates_from_html(self, html_file: Path, update: bool = False) -> Set[str]:
+        logger.info(f"Getting certificate ids from {html_file}")
         with open(html_file, "r", encoding="utf-8") as handle:
-            soup = BeautifulSoup(handle.read(), 'html5lib')
+            html = BeautifulSoup(handle.read(), 'html5lib')
 
-        tables = soup.find_all('table', id='searchResultsTable')
-        assert len(tables) == 1
-
-        for row in tables[0].tbody.find_all('tr'):
-            cert_id = row.find("a").text
-            if cert_id not in self.certs:
-                self.certs[cert_id] = None
+        table = [x for x in html.find(id="searchResultsTable").tbody.contents if x != "\n"]
+        entries: Set[str] = set()
+        
+        for entry in table:
+            if isinstance(entry, NavigableString):
+                continue
+            cert_id = entry.find("a").text
+            if cert_id not in entries:
+                entries.add(cert_id)
+        
+        return entries
 
     @serialize
-    def web_scan(self, redo: bool = False):
+    def web_scan(self, cert_ids: Set[str], redo: bool = False):
         logger.info("Entering web scan.")
-        for cert_id, cert in self.certs.items():
+        for cert_id in cert_ids:
             self.certs[cert_id] = FIPSCertificate.html_from_file(
                 self.web_dir / f"{cert_id}.html",
                 FIPSCertificate.State(
@@ -202,7 +211,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                     None,
                     False,
                 ),
-                cert,
+                self.certs[cert_id] if cert_id in self.certs else None,
                 redo=redo,
             )
 
@@ -222,16 +231,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         cert: FIPSCertificate
         for cert in self.certs.values():
             cert.set_local_paths(self.policies_dir, self.web_dir, self.fragments_dir)
-
-    def _append_new_certs_data(self) -> int:
-        # we need to know the exact certificates downloaded, so we don't overwrite something already done
-        new_certs = self.download_all_htmls()
-        self.download_all_pdfs()
-        logger.info(f"{self.new_files} needed to be downloaded")
-        for cert_id in new_certs:
-            self.certs[cert_id] = None
-        return len(new_certs)
-
+            
     @serialize
     def get_certs_from_web(
         self,
@@ -257,10 +257,10 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         self.algs_dir.mkdir(exist_ok=True)
 
         # Download files containing all available module certs (always)
-        self.prepare_dataset(test, update)
+        cert_ids = self.prepare_dataset(test, update)
 
-        logger.info("Downloading sample html and security policies")
-        self.download_neccessary_files()
+        logger.info("Downloading certificate html and security policies")
+        self.download_neccessary_files(cert_ids)
 
         if not no_download_algorithms:
             aset = FIPSAlgorithmDataset({}, Path(self.root_dir / 'web' / 'algorithms'), 'algorithms', 'sample algs')
@@ -269,7 +269,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
             self.algorithms = aset
         
-        self.web_scan(redo=redo_web_scan)
+        self.web_scan(cert_ids, redo=redo_web_scan)
 
     @serialize
     def deprocess(self):
@@ -294,7 +294,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
             [
                 (cert, high_precision)
                 for cert in self.certs.values()
-                if (not cert.state.tables_done or high_precision) and cert.state.txt_state
+                if cert is not None and (not cert.state.tables_done or high_precision) and cert.state.txt_state
             ],
             config.n_threads // 4,  # tabula already processes by parallel, so
             # it's counterproductive to use all threads
@@ -304,8 +304,9 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
         not_decoded = [cert.state.sp_path for done, cert, _ in result if done is False]
         for state, cert, algorithms in result:
-            self.certs[cert.dgst].state.tables_done = state
-            self.certs[cert.dgst].pdf_scan.algorithms += algorithms
+            certificate = self.certs[cert.dgst]        
+            certificate.state.tables_done = state
+            certificate.pdf_scan.algorithms += algorithms
         return not_decoded
 
     def remove_algorithms_from_extracted_data(self):
@@ -331,18 +332,29 @@ class FIPSDataset(Dataset, ComplexSerializableType):
             # returns True if candidates should _not_ be matched
 
     def _compare_certs(self, current_certificate: "FIPSCertificate", other_id: str):
-        cert_first = current_certificate.web_scan.date_validation[0].year
-        cert_last = current_certificate.web_scan.date_validation[-1].year
-        conn_first = self.certs[other_id].web_scan.date_validation[0].year
-        conn_last = self.certs[other_id].web_scan.date_validation[-1].year
+        other_cert = self.certs[other_id]
+        if current_certificate.web_scan.date_validation is None \
+            or other_cert is None or other_cert.web_scan.date_validation is None:
+                raise RuntimeError("Building of the dataset probably failed - this should not be happening.")
+            
+        cert_first = current_certificate.web_scan.date_validation[0]
+        cert_last = current_certificate.web_scan.date_validation[-1]
+        conn_first = other_cert.web_scan.date_validation[0]
+        conn_last = other_cert.web_scan.date_validation[-1]
+        
+        if not isinstance(cert_first, datetime.date) or not isinstance(cert_last, datetime.date)\
+            or not isinstance(conn_first, datetime.date) or not isinstance(conn_last, datetime.date):
+                raise RuntimeError("Dataset was probably not built correctly - this should not be happening.")
 
         return (
-            cert_first - conn_first > config.year_difference_between_validations
-            and cert_last - conn_last > config.year_difference_between_validations
-            or cert_first < conn_first
+            cert_first.year - conn_first.year > config.year_difference_between_validations
+            and cert_last.year - conn_last.year > config.year_difference_between_validations
+            or cert_first.year < conn_first.year
         )
 
     def _remove_false_positives_for_cert(self, current_cert: FIPSCertificate):
+        if current_cert.heuristics.keywords is None:
+            raise RuntimeError("Dataset was probably not built correctly - this should not be happening.")
         for rule in current_cert.heuristics.keywords["rules_cert_id"]:
             matches = current_cert.heuristics.keywords["rules_cert_id"][rule]
             current_cert.heuristics.keywords["rules_cert_id"][rule] = [
@@ -361,6 +373,10 @@ class FIPSDataset(Dataset, ComplexSerializableType):
             processed_cert, cert_candidate
         ):
             return False
+        
+        if self.algorithms is None:
+            raise RuntimeError("Dataset was probably not built correctly - this should not be happening.")
+        
         if cert_candidate not in self.algorithms.certs:
             return True
 
@@ -372,6 +388,9 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
         algs = self.algorithms.certs[cert_candidate]
         for current_alg in algs:
+            if current_alg.vendor is None or processed_cert.web_scan.vendor is None:
+                raise RuntimeError("Dataset was probably not built correctly - this should not be happening.")
+
             if FIPSCertificate.get_compare(processed_cert.web_scan.vendor) == FIPSCertificate.get_compare(
                 current_alg.vendor
             ):
@@ -417,39 +436,42 @@ class FIPSDataset(Dataset, ComplexSerializableType):
             FIPSDataset._find_connections(current_cert)
 
     @serialize
-    def finalize_results(self, use_nist_cpe_matching_dict: bool = True):
+    def finalize_results(self, use_nist_cpe_matching_dict: bool = True, perform_cpe_heuristics: bool = True):
         logger.info("Entering 'analysis' and building connections between certificates.")
         self.unify_algorithms()
         self.remove_algorithms_from_extracted_data()
         self.validate_results()
-
-        self.compute_cpe_heuristics()
-        self.compute_related_cves(use_nist_cpe_matching_dict=use_nist_cpe_matching_dict)
+        if perform_cpe_heuristics:
+            self.compute_cpe_heuristics()
+            self.compute_related_cves(use_nist_cpe_matching_dict=use_nist_cpe_matching_dict)
 
     def _highlight_vendor_in_dot(self, dot: Digraph, current_key: str, highlighted_vendor: str):
-        if self.certs[current_key].web_scan.vendor != highlighted_vendor:
+        current_cert = self.certs[current_key]
+        
+        if current_cert.web_scan.vendor != highlighted_vendor:
             return
 
         dot.attr("node", color="red")
-        if self.certs[current_key].web_scan.status == "Revoked":
+        if current_cert.web_scan.status == "Revoked":
             dot.attr("node", color="grey32")
-        if self.certs[current_key].web_scan.status == "Historical":
+        if current_cert.web_scan.status == "Historical":
             dot.attr("node", color="gold3")
 
     def _add_colored_node(self, dot: Digraph, current_key: str, highlighted_vendor: str):
+        current_cert = self.certs[current_key]
         dot.attr("node", color="lightgreen")
-        if self.certs[current_key].web_scan.status == "Revoked":
+        if current_cert.web_scan.status == "Revoked":
             dot.attr("node", color="lightgrey")
-        if self.certs[current_key].web_scan.status == "Historical":
+        if current_cert.web_scan.status == "Historical":
             dot.attr("node", color="gold")
         self._highlight_vendor_in_dot(dot, current_key, highlighted_vendor)
         dot.node(
             current_key,
             label=current_key
             + "&#10;"
-            + self.certs[current_key].web_scan.vendor
+            + current_cert.web_scan.vendor if current_cert.web_scan.vendor is not None else ""
             + "&#10;"
-            + (self.certs[current_key].web_scan.module_name if self.certs[current_key].web_scan.module_name else ""),
+            + (current_cert.web_scan.module_name if current_cert.web_scan.module_name else ""),
         )
 
     def _get_processed_list(self, connection_list: str, key: str):
@@ -484,7 +506,9 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         edges = 0
 
         for key in self.certs:
-            if key == "Not found" or not self.certs[key].state.file_status:
+            cert = self.certs[key]
+            
+            if key == "Not found" or not cert.state.file_status:
                 continue
 
             processed = self._get_processed_list(connection_list, key)
@@ -499,12 +523,14 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                     key,
                     label=key
                     + "\r\n"
-                    + self.certs[key].web_scan.vendor
-                    + ("\r\n" + self.certs[key].web_scan.module_name if self.certs[key].web_scan.module_name else ""),
+                    + cert.web_scan.vendor if cert.web_scan.vendor is not None else ""
+                    + ("\r\n" + cert.web_scan.module_name if cert.web_scan.module_name else ""),
                 )
 
         for key in self.certs:
-            if key == "Not found" or not self.certs[key].state.file_status:
+            cert = self.certs[key]
+            
+            if key == "Not found" or not cert.state.file_status:
                 continue
             processed = self._get_processed_list(connection_list, key)
             for conn in processed:
@@ -535,9 +561,9 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
     def group_vendors(self) -> Dict:
         vendors = {}
-        v = {x.web_scan.vendor.lower() for x in self.certs.values()}
-        v = sorted(v, key=FIPSCertificate.get_compare)
-        for prefix, a in groupby(v, key=FIPSCertificate.get_compare):
+        v = {x.web_scan.vendor.lower() for x in self.certs.values() if x is not None and x.web_scan.vendor is not None}
+        v_sorted = sorted(v, key=FIPSCertificate.get_compare)
+        for prefix, a in groupby(v_sorted, key=FIPSCertificate.get_compare):
             vendors[prefix] = list(a)
 
         return vendors
diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py
index 11f3d6c3..860981f3 100644
--- a/sec_certs/dataset/fips_algorithm.py
+++ b/sec_certs/dataset/fips_algorithm.py
@@ -17,6 +17,8 @@ logger = logging.getLogger(__name__)
 
 
 class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
+    
+    certs: Dict[str, List] # type: ignore # noqa
     def get_certs_from_web(self):
         self.root_dir.mkdir(exist_ok=True)
         algs_paths, algs_urls = [], []
@@ -111,7 +113,9 @@ class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
     def to_dict(self):
         return self.__dict__ 
 
-    def to_json(self, output_path: Union[str, Path]):
+    def to_json(self, output_path: Union[str, Path] = None):
+        if not output_path:
+            output_path = self.json_path
         with Path(output_path).open('w') as handle:
             json.dump(self, handle, indent=4, cls=CustomJSONEncoder)
 
diff --git a/sec_certs/dataset/protection_profile.py b/sec_certs/dataset/protection_profile.py
index 558faa23..eae3cb64 100644
--- a/sec_certs/dataset/protection_profile.py
+++ b/sec_certs/dataset/protection_profile.py
@@ -47,7 +47,7 @@ class ProtectionProfileDataset:
         return cls(dct)
 
     @classmethod
-    def from_web(cls, store_dataset_path: Optional[Union[str, Path]]):
+    def from_web(cls, store_dataset_path: Optional[Path]):
         logger.info(f'Downloading static PP dataset from: {cls.static_dataset_url}')
         if not store_dataset_path:
             tmp = tempfile.TemporaryDirectory()
diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py
index aa5f8dd7..9884a071 100644
--- a/sec_certs/helpers.py
+++ b/sec_certs/helpers.py
@@ -14,6 +14,10 @@ from datetime import date
 import numpy as np
 import pandas as pd
 import subprocess
+import time
+import copy
+from packaging.version import VERSION_PATTERN
+
 
 
 from enum import Enum
@@ -30,15 +34,14 @@ from sec_certs.constants import TAG_MATCH_COUNTER, APPEND_DETAILED_MATCH_MATCHES
 
 logger = logging.getLogger(__name__)
 
-
 def tqdm(*args, **kwargs):
     if "disable" in kwargs:
         return tqdm_original(*args, **kwargs)
     return tqdm_original(*args, **kwargs, disable=not config.enable_progress_bars)
 
-
-def download_file(url: str, output: Path) -> int:
+def download_file(url: str, output: Path, delay: float = 0) -> Union[str, int]:
     try:
+        time.sleep(delay)
         r = requests.get(url, allow_redirects=True, timeout=constants.REQUEST_TIMEOUT)
         if r.status_code == requests.codes.ok:
             with output.open("wb") as f:
@@ -79,7 +82,7 @@ def get_sha256_filepath(filepath):
     return hash_sha256.hexdigest()
 
 
-def sanitize_link(record: str) -> Union[str, None]:
+def sanitize_link(record: Optional[str]) -> Optional[str]:
     if not record:
         return None
     return record.replace(':443', '').replace(' ', '%20').replace('http://', 'https://')
@@ -94,7 +97,7 @@ def sanitize_date(record: Union[pd.Timestamp, date, np.datetime64]) -> Union[dat
         return record
 
 
-def sanitize_string(record: str) -> Union[str, None]:
+def sanitize_string(record: Optional[str]) -> Optional[str]:
     if not record:
         return None
     else:
@@ -176,8 +179,8 @@ def find_tables(txt: str, file_name: Path) -> Optional[List]:
 
     # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer
     logger.info(f'parsing tables in {file_name}')
-    rb = find_tables_iterative(txt)
-    return rb if rb else None
+    table_page_indices = find_tables_iterative(txt)
+    return table_page_indices if table_page_indices else None
 
 
 def repair_pdf(file: Path):
@@ -325,7 +328,7 @@ def search_only_headers_anssi(filepath: Path):
     for rule in rules_certificate_preface:
         num_rules_hits[rule[1]] = 0
 
-    items_found = {}
+    items_found = {} # type: ignore # noqa
 
     try:
         whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath)
@@ -416,7 +419,7 @@ def search_only_headers_bsi(filepath: Path):
         '(BSI-DSZ-CC-.+?) zu (.+?) der (.*)',
     ]
 
-    items_found = {}
+    items_found = {} # type: ignore # noqa
     no_match_yet = True
 
     try:
@@ -499,7 +502,7 @@ def search_only_headers_bsi(filepath: Path):
     return constants.RETURNCODE_OK, items_found
 
 
-def extract_keywords(filepath: Path) -> Tuple[int, Optional[Dict[str, Dict[str, int]]]]:
+def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, int]]]]:
     try:
         result = parse_cert_file(filepath, cc_search_rules, -1, sec_certs.constants.LINE_SEPARATOR)[0]
 
@@ -728,9 +731,13 @@ def compute_heuristics_version(cert_name: str) -> List[str]:
     # identified_versions = list(set([max(x, key=len) for x in re.findall(VERSION_PATTERN, cert_name, re.IGNORECASE | re.VERBOSE)]))
     # return identified_versions if identified_versions else ['-']
 
-    return [re.search(normalizer, x).group() for x in matched_strings] if matched_strings else ['-']
+    if not matched_strings:
+        return ['-']
+
+    matched = [re.search(normalizer, x) for x in matched_strings]
+    return [x.group() for x in matched if x is not None]
 
-def tokenize_dataset(dset: List[str], keywords: Set[str]) -> np.array:
+def tokenize_dataset(dset: List[str], keywords: Set[str]) -> np.ndarray: 
     return np.array([tokenize(x, keywords) for x in dset])
 
 def tokenize(string: str, keywords: Set[str]) -> str:
diff --git a/sec_certs/model/cpe_matching.py b/sec_certs/model/cpe_matching.py
index 1aba8802..81fc8206 100644
--- a/sec_certs/model/cpe_matching.py
+++ b/sec_certs/model/cpe_matching.py
@@ -44,7 +44,7 @@ class CPEClassifier(BaseEstimator):
         @param cpes: List of CPEs to filtered
         @return All CPEs in cpes variable which item name has at least 4 characters.
         """
-        return list(filter(lambda x: len(x.item_name) > 3, cpes))
+        return list(filter(lambda x: x.item_name is not None and len(x.item_name) > 3, cpes))
 
     def build_lookup_structures(self, X: List[CPE]):
         """
@@ -101,11 +101,15 @@ class CPEClassifier(BaseEstimator):
         sanitized_product_name = CPEClassifier._fully_sanitize_string(product_name) if product_name else product_name
         candidate_vendors = self.get_candidate_list_of_vendors(sanitized_vendor)
 
+        if candidate_vendors is None:
+            raise RuntimeError("Candidate vendors were not calculated successfully.")
+        if versions is None:
+            raise RuntimeError("Version were not calculated successfully.")
         candidates = self.get_candidate_cpe_matches(candidate_vendors, versions)
         ratings = [self.compute_best_match(cpe, sanitized_product_name, candidate_vendors, versions, relax_title=relax_title) for cpe in candidates]
         threshold = self.match_threshold if not relax_version else 100
-        final_matches = list(filter(lambda x: x[0] >= threshold, zip(ratings, candidates)))
-        final_matches = [x[1].uri for x in final_matches[:self.n_max_matches]]
+        final_matches_aux: List[Tuple[float, CPE]] = list(filter(lambda x: x[0] >= threshold, zip(ratings, candidates)))
+        final_matches: Optional[List[str]] = [x[1].uri for x in final_matches_aux[:self.n_max_matches] if x[1].uri is not None]
 
 
 
@@ -129,6 +133,8 @@ class CPEClassifier(BaseEstimator):
         @return: Maximal value of the four string similarities discussed above.
         """
         if relax_title:
+            if cpe.title is None or cpe.vendor is None or cpe.item_name is None or cpe.version is None or cpe.update is None or cpe.target_hw is None:
+                raise RuntimeError(f"There was a problem in computing best match for CPE {cpe.title if cpe.title else cpe.item_name}")
             sanitized_title = CPEClassifier._fully_sanitize_string(cpe.title) if cpe.title else CPEClassifier._fully_sanitize_string(cpe.vendor + ' ' + cpe.item_name + ' ' + cpe.version + ' ' + cpe.update + ' ' + cpe.target_hw)
         else:
             if cpe.title:
@@ -136,6 +142,8 @@ class CPEClassifier(BaseEstimator):
             else:
                 return 0
 
+        if cpe.item_name is None or versions is None:
+                raise RuntimeError(f"There was a problem in computing best match for CPE {cpe.title}")
         sanitized_item_name = CPEClassifier._fully_sanitize_string(cpe.item_name)
         cert_stripped = CPEClassifier._strip_manufacturer_and_version(product_name, candidate_vendors, versions)
 
@@ -177,14 +185,14 @@ class CPEClassifier(BaseEstimator):
         if not manufacturer:
             return None
 
-        result = set()
+        result: Set = set()
         splits = re.compile(r'[,/]').findall(manufacturer)
 
         if splits:
             vendor_tokens = set(itertools.chain.from_iterable([[x.strip() for x in manufacturer.split(s)] for s in splits]))
-            result = [self.get_candidate_list_of_vendors(x) for x in vendor_tokens]
-            result = list(set(itertools.chain.from_iterable([x for x in result if x])))
-            return result if result else None
+            result_aux = [self.get_candidate_list_of_vendors(x) for x in vendor_tokens]
+            result_used = list(set(itertools.chain.from_iterable([x for x in result_aux if x])))
+            return result_used if result_used else None
 
         if manufacturer in self.vendors_:
             result.add(manufacturer)
@@ -206,7 +214,8 @@ class CPEClassifier(BaseEstimator):
         if 'athena' in tokenized and 'smartcard' in tokenized:
             result.add('athena-scs')
         if tokenized[0] == 'the' and not result:
-            result = self.get_candidate_list_of_vendors(' '.join(tokenized[1:]))
+            candidate_result = self.get_candidate_list_of_vendors(' '.join(tokenized[1:]))
+            return list(candidate_result) if candidate_result else None
 
         return list(result) if result else None
 
diff --git a/sec_certs/model/dependency_finder.py b/sec_certs/model/dependency_finder.py
index 7c0d151f..75d6ac4a 100644
--- a/sec_certs/model/dependency_finder.py
+++ b/sec_certs/model/dependency_finder.py
@@ -111,13 +111,17 @@ class DependencyFinder:
                 DependencyFinder._get_affecting_indirectly(cert_id, referenced_by_indirect)
 
     def get_directly_affected_by(self, dgst: str) -> Optional[List[str]]:
-        return self.dependencies[dgst].get("directly_affected_by", None)
+        res = self.dependencies[dgst].get("directly_affected_by", None)
+        return list(res) if res else None
 
     def get_indirectly_affected_by(self, dgst: str) -> Optional[Set[str]]:
-        return self.dependencies[dgst].get("indirectly_affected_by", None)
+        res = self.dependencies[dgst].get("indirectly_affected_by", None)
+        return set(res) if res else None
 
     def get_directly_affecting(self, dgst: str) -> Optional[Set[str]]:
-        return self.dependencies[dgst].get("directly_affecting", None)
+        res = self.dependencies[dgst].get("directly_affecting", None)
+        return set(res) if res else None
 
     def get_indirectly_affecting(self, dgst: str) -> Optional[Set[str]]:
-        return self.dependencies[dgst].get("indirectly_affecting", None)
+        res = self.dependencies[dgst].get("indirectly_affecting", None)
+        return set(res) if res else None
diff --git a/sec_certs/model/evaluation.py b/sec_certs/model/evaluation.py
index 7bfa337c..f0c98975 100644
--- a/sec_certs/model/evaluation.py
+++ b/sec_certs/model/evaluation.py
@@ -19,15 +19,15 @@ def get_validation_dgsts(filepath: Union[str, Path]) -> Set[str]:
         return set(json.load(handle))
 
 
-def compute_precision(y: np.array, y_pred: np.array, **kwargs):
+def compute_precision(y: np.ndarray, y_pred: np.ndarray, **kwargs):
     prec = []
     for true, pred in zip(y, y_pred):
         set_pred = set(pred) if pred else set()
         set_true = set(true) if true else set()
         if set_pred and not set_true:
-            prec.append(0)
+            prec.append(0.0)
         elif not set_pred and not set_true:
-            prec.append(1)
+            prec.append(1.0)
         else:
             prec.append(len(set_true.intersection(set_pred)) / len(set_true))
     return np.mean(prec)
@@ -35,7 +35,7 @@ def compute_precision(y: np.array, y_pred: np.array, **kwargs):
 
 def evaluate(x_valid: List[Union[CommonCriteriaCert, FIPSCertificate]], y_valid: List[Optional[List[str]]], outpath: Optional[Union[Path, str]], cpe_dset: CPEDataset):
     y_pred = [x.heuristics.cpe_matches for x in x_valid]
-    precision = compute_precision(y_valid, y_pred)
+    precision = compute_precision(np.array(y_valid), np.array(y_pred))
 
     correctly_classified = []
     badly_classified = []
diff --git a/sec_certs/parallel_processing.py b/sec_certs/parallel_processing.py
index 2134e924..ae8a1221 100644
--- a/sec_certs/parallel_processing.py
+++ b/sec_certs/parallel_processing.py
@@ -1,6 +1,6 @@
 from sec_certs.helpers import tqdm
 from multiprocessing.pool import Pool, ThreadPool
-from typing import Callable, Iterable, Optional
+from typing import Callable, Iterable, Optional, Union
 import time
 
 
@@ -8,7 +8,7 @@ def process_parallel(func: Callable, items: Iterable, max_workers: int, callback
                      use_threading: bool = True, progress_bar: bool = True, unpack: bool = False,
                      progress_bar_desc: Optional[str] = None):
 
-    pool = ThreadPool(max_workers) if use_threading else Pool(max_workers)
+    pool: Union[Pool, ThreadPool] = ThreadPool(max_workers) if use_threading else Pool(max_workers)
     results = [pool.apply_async(func, (*i,), callback=callback) for i in items] if unpack else [pool.apply_async(func, (i, ), callback=callback) for i in items]
 
     if progress_bar is True and items:
diff --git a/sec_certs/sample/cc_maintenance_update.py b/sec_certs/sample/cc_maintenance_update.py
index 8333accd..6fce7ebb 100644
--- a/sec_certs/sample/cc_maintenance_update.py
+++ b/sec_certs/sample/cc_maintenance_update.py
@@ -44,5 +44,11 @@ class CommonCriteriaMaintenanceUpdate(CommonCriteriaCert, ComplexSerializableTyp
 
     @classmethod
     def get_updates_from_cc_cert(cls, cert: CommonCriteriaCert):
-        return [cls(x.maintenance_title, x.maintenance_report_link, x.maintenance_st_link,
-                    None, None, None, cert.dgst, x.maintenance_date) for x in cert.maintenance_updates]
+        if cert.maintenance_updates is None:
+            raise RuntimeError("Dataset was probably not built correctly - this should not be happening.")
+        
+        return [cls(x.maintaeance_title, x.maintenance_report_link, x.maintenance_st_link,
+                    None, None, None, cert.dgst, x.maintenance_date) for x in cert.maintenance_updates
+                if (x.maintenance_title is not None and x.maintenance_report_link is not None\
+                    and x.maintenance_st_link is not None and x.maintenance_date is not None)]
+
diff --git a/sec_certs/sample/certificate.py b/sec_certs/sample/certificate.py
index 9830f656..b0522d67 100644
--- a/sec_certs/sample/certificate.py
+++ b/sec_certs/sample/certificate.py
@@ -5,7 +5,7 @@ import json
 import itertools
 
 from abc import ABC, abstractmethod
-from typing import Union, TypeVar, Type, Any
+from typing import Optional, Union, TypeVar, Type, Any
 
 from sec_certs.serialization.json import CustomJSONDecoder, CustomJSONEncoder, ComplexSerializableType
 from sec_certs.model.cpe_matching import CPEClassifier
@@ -38,7 +38,9 @@ class Certificate(ABC, ComplexSerializableType):
     def label_studio_title(self):
         raise NotImplementedError('Not meant to be implemented')
 
-    def __eq__(self, other: 'Certificate') -> bool:
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Certificate):
+            return NotImplemented
         return self.dgst == other.dgst
 
     def to_dict(self):
@@ -49,7 +51,9 @@ class Certificate(ABC, ComplexSerializableType):
         dct.pop('dgst')
         return cls(*(tuple(dct.values())))
 
-    def to_json(self, output_path: Union[Path, str]):
+    def to_json(self, output_path: Optional[Union[str, Path]] = None):
+        if output_path is None:
+            raise RuntimeError(f"You tried to serialize an object ({type(self)}) that does not have implicit json path. Please provide json_path.")
         with Path(output_path).open('w') as handle:
             json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
 
@@ -75,6 +79,6 @@ class Certificate(ABC, ComplexSerializableType):
             related_cves = [cve_dataset.get_cve_ids_for_cpe_uri(x) for x in self.heuristics.cpe_matches]
             related_cves = list(filter(lambda x: x is not None, related_cves))
             if related_cves:
-                self.heuristics.related_cves = set(itertools.chain.from_iterable(related_cves))
+                self.heuristics.related_cves = set(itertools.chain.from_iterable([x for x in related_cves if x is not None]))
         else:
             self.heuristics.related_cves = None
diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py
index 3e25f35a..890c38e8 100644
--- a/sec_certs/sample/common_criteria.py
+++ b/sec_certs/sample/common_criteria.py
@@ -3,7 +3,7 @@ import operator
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path
-from typing import Optional, List, Dict, Union, Any, Set, ClassVar
+from typing import Optional, List, Dict, Tuple, Union, Any, Set, ClassVar
 
 
 import requests
@@ -26,10 +26,10 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
         """
         Object for holding maintenance reports.
         """
-        maintenance_date: date
-        maintenance_title: str
-        maintenance_report_link: str
-        maintenance_st_link: str
+        maintenance_date: Optional[date]
+        maintenance_title: Optional[str]
+        maintenance_report_link: Optional[str]
+        maintenance_st_link: Optional[str]
 
         def __post_init__(self):
             super().__setattr__('maintenance_report_link',
@@ -173,7 +173,7 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
 
     @dataclass
     class CCHeuristics(ComplexSerializableType):
-        extracted_versions: List[str] = field(default=None)
+        extracted_versions: Optional[List[str]] = field(default=None)
         cpe_matches: Optional[Set[str]] = field(default=None)
         verified_cpe_matches: Optional[Set[str]] = field(default=None)
         related_cves: Optional[Set[str]] = field(default=None)
@@ -203,12 +203,12 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
                                         'verified_cpe_matches', 'related_cves', 'directly_affected_by',
                                         'indirectly_affected_by', 'directly_affecting', 'indirectly_affecting']
 
-    def __init__(self, status: str, category: str, name: str, manufacturer: str, scheme: str,
-                 security_level: Union[str, set], not_valid_before: date,
-                 not_valid_after: date, report_link: str, st_link: str, cert_link: Optional[str],
+    def __init__(self, status: str, category: str, name: str, manufacturer: Optional[str], scheme: str,
+                 security_level: Union[str, set], not_valid_before: Optional[date],
+                 not_valid_after: Optional[date], report_link: str, st_link: str, cert_link: Optional[str],
                  manufacturer_web: Optional[str],
-                 protection_profiles: Set[ProtectionProfile],
-                 maintenance_updates: Set[MaintenanceReport],
+                 protection_profiles: Optional[Set[ProtectionProfile]],
+                 maintenance_updates: Optional[Set[MaintenanceReport]],
                  state: Optional[InternalState],
                  pdf_data: Optional[PdfData],
                  heuristics: Optional[CCHeuristics]):
@@ -246,6 +246,8 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
         """
         Computes the primary key of the sample using first 16 bytes of SHA-256 digest
         """
+        if not (self.name is not None and self.report_link is not None and self.category is not None):
+            raise RuntimeError("Certificate digest can't be computed, because information is missing.")
         return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link)
 
     @property
@@ -329,13 +331,13 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
                     protection_profiles.add(ProtectionProfile(str(link.contents[0]), CommonCriteriaCert.cc_url + link.get('href')))
             return protection_profiles
 
-        def _get_date(cell: Tag) -> date:
+        def _get_date(cell: Tag) -> Optional[date]:
             text = cell.get_text()
             extracted_date = datetime.strptime(
                 text, '%Y-%m-%d').date() if text else None
             return extracted_date
 
-        def _get_report_st_links(cell: Tag) -> [str, str]:
+        def _get_report_st_links(cell: Tag) -> Tuple[str, str]:
             links = cell.find_all('a')
             # TODO: Exception checks
             assert links[1].get('title').startswith('Certification Report')
@@ -422,6 +424,7 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
 
     @staticmethod
     def download_pdf_report(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code: Union[str, int]
         if not cert.report_link:
             exit_code = 'No link'
         else:
@@ -430,11 +433,14 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
             error_msg = f'failed to download report from {cert.report_link}, code: {exit_code}'
             logger.error(f'Cert dgst: {cert.dgst} ' + error_msg)
             cert.state.report_download_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(error_msg)
         return cert
 
     @staticmethod
     def download_pdf_target(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code: Union[str, int]
         if not cert.st_link:
             exit_code = 'No link'
         else:
@@ -443,6 +449,8 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
             error_msg = f'failed to download ST from {cert.report_link}, code: {exit_code}'
             logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
             cert.state.st_download_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(error_msg)
         return cert
 
@@ -453,6 +461,8 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
             error_msg = 'failed to convert report pdf->txt'
             logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
             cert.state.report_convert_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(error_msg)
         return cert
 
@@ -463,6 +473,8 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
             error_msg = 'failed to convert security target pdf->txt'
             logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
             cert.state.st_convert_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(error_msg)
         return cert
 
@@ -471,6 +483,8 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
         response, cert.pdf_data.st_metadata = helpers.extract_pdf_metadata(cert.state.st_pdf_path)
         if response != constants.RETURNCODE_OK:
             cert.state.st_extract_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(response)
         return cert
 
@@ -479,6 +493,8 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
         response, cert.pdf_data.report_metadata = helpers.extract_pdf_metadata(cert.state.report_pdf_path)
         if response != constants.RETURNCODE_OK:
             cert.state.report_extract_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(response)
         return cert
 
@@ -491,9 +507,13 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
 
         if response_anssi != constants.RETURNCODE_OK:
             cert.state.st_extract_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(response_anssi)
         if response_bsi != constants.RETURNCODE_OK:
             cert.state.st_extract_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(response_bsi)
 
         return cert
@@ -508,9 +528,13 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
 
         if response_anssi != constants.RETURNCODE_OK:
             cert.state.report_extract_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(response_anssi)
         if response_bsi != constants.RETURNCODE_OK:
             cert.state.report_extract_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(response_bsi)
 
         return cert
@@ -527,6 +551,8 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
         response, cert.pdf_data.st_keywords = helpers.extract_keywords(cert.state.st_txt_path)
         if response != constants.RETURNCODE_OK:
             cert.state.st_extract_ok = False
+            if not cert.state.errors:
+                cert.state.errors = []
             cert.state.errors.append(response)
         return cert
 
@@ -535,10 +561,10 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl
 
     def compute_heuristics_cpe_vendors(self, cpe_classifier: CPEClassifier):
         # TODO: This method probably can be deleted.
-        self.heuristics.cpe_candidate_vendors = cpe_classifier.get_candidate_list_of_vendors(self.manufacturer)
+        self.heuristics.cpe_candidate_vendors = cpe_classifier.get_candidate_list_of_vendors(self.manufacturer) # type: ignore
 
     def compute_heuristics_cpe_match(self, cpe_classifier: CPEClassifier):
-        self.heuristics.cpe_matches = cpe_classifier.predict_single_cert(self.manufacturer, self.name, self.heuristics.extracted_versions)
+        self.heuristics.cpe_matches = cpe_classifier.predict_single_cert(self.manufacturer, self.name, self.heuristics.extracted_versions)  # type: ignore
 
     def compute_heuristics_cert_lab(self):
         if not self.pdf_data:
diff --git a/sec_certs/sample/cpe.py b/sec_certs/sample/cpe.py
index d70f2783..c2b542ce 100644
--- a/sec_certs/sample/cpe.py
+++ b/sec_certs/sample/cpe.py
@@ -6,11 +6,11 @@ from sec_certs.serialization.pandas import PandasSerializableType
 
 @dataclass(init=False)
 class CPE(PandasSerializableType, ComplexSerializableType):
-    uri: str
-    title: str
-    version: str
-    vendor: str
-    item_name: str
+    uri: Optional[str]
+    title: Optional[str]
+    version: Optional[str]
+    vendor: Optional[str]
+    item_name: Optional[str]
     start_version: Optional[Tuple[str, str]]
     end_version: Optional[Tuple[str, str]]
 
@@ -22,8 +22,8 @@ class CPE(PandasSerializableType, ComplexSerializableType):
                  end_version: Optional[Tuple[str, str]] = None):
         self.uri = uri
         self.title = title
-        self.start_version = tuple(start_version) if start_version else None
-        self.end_version = tuple(end_version) if end_version else None
+        self.start_version = start_version if start_version else None
+        self.end_version = end_version if end_version else None
 
         if self.uri:
             self.vendor = ' '.join(self.uri.split(':')[3].split('_'))
@@ -31,6 +31,8 @@ class CPE(PandasSerializableType, ComplexSerializableType):
             self.version = self.uri.split(':')[5]
 
     def __lt__(self, other: 'CPE'):
+        if self.title is None or other.title is None:
+            raise RuntimeError("Cannot compare CPEs because title is missing.")
         return self.title < other.title
 
     @property
@@ -39,10 +41,14 @@ class CPE(PandasSerializableType, ComplexSerializableType):
 
     @property
     def update(self) -> str:
+        if self.uri is None:
+            raise RuntimeError("URI is missing.")
         return ' '.join(self.uri.split(':')[6].split('_'))
 
     @property
     def target_hw(self) -> str:
+        if self.uri is None:
+            raise RuntimeError("URI is missing.")
         return ' '.join(self.uri.split(':')[11].split('_'))
 
     @property
diff --git a/sec_certs/sample/cve.py b/sec_certs/sample/cve.py
index 0c125ad9..f63a8869 100644
--- a/sec_certs/sample/cve.py
+++ b/sec_certs/sample/cve.py
@@ -1,7 +1,7 @@
 import datetime
 import itertools
 from dataclasses import dataclass
-from typing import Dict, List, Optional, ClassVar
+from typing import Any, Dict, List, Optional, ClassVar, Tuple
 
 from dateutil.parser import isoparse
 
@@ -90,7 +90,8 @@ class CVE(PandasSerializableType, ComplexSerializableType):
                     for x in lst:
                         if x['vulnerable']:
                             cpe_uri = x['cpe23Uri']
-
+                            version_start: Optional[Tuple[str, str]]
+                            version_end: Optional[Tuple[str, str]]
                             if 'versionStartIncluding' in x and x['versionStartIncluding']:
                                 version_start = ('including', x['versionStartIncluding'])
                             elif 'versionStartExcluding' in x and x['versionStartExcluding']:
diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py
index 3cd8b38e..a8842e00 100644
--- a/sec_certs/sample/fips.py
+++ b/sec_certs/sample/fips.py
@@ -151,16 +151,16 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         connections: List[str]
         unmatched_algs: int
 
-        extracted_versions: List[str] = field(default=None)
+        extracted_versions: Optional[List[str]] = field(default=None)
         cpe_matches: Optional[Set[str]] = field(default=None)
         verified_cpe_matches: Optional[Set[CPE]] = field(default=None)
         related_cves: Optional[List[str]] = field(default=None)
         cpe_candidate_vendors: Optional[List[str]] = field(init=False)
 
-        directly_affected_by: Set = field(default=None)
-        indirectly_affected_by: Set = field(default=None)
-        directly_affecting: Set = field(default=None)
-        indirectly_affecting: Set = field(default=None)
+        directly_affected_by: Optional[Set] = field(default=None)
+        indirectly_affected_by: Optional[Set] = field(default=None)
+        directly_affecting: Optional[Set] = field(default=None)
+        indirectly_affecting: Optional[Set] = field(default=None)
 
         @property
         def serialized_attributes(self) -> List[str]:
@@ -193,7 +193,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
     @staticmethod
     def download_security_policy(cert: Tuple[str, Path]) -> None:
-        exit_code = helpers.download_file(*cert)
+        exit_code = helpers.download_file(*cert, delay=1)
         if exit_code != requests.codes.ok:
             logger.error(
                 f'Failed to download security policy from {cert[0]}, code: {exit_code}')
@@ -212,7 +212,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
     @staticmethod
     def download_html_page(cert: Tuple[str, Path]) -> Optional[Tuple[str, Path]]:
-        exit_code = helpers.download_file(*cert)
+        exit_code = helpers.download_file(*cert, delay=1)
         if exit_code != requests.codes.ok:
             logger.error(
                 f'Failed to download html page from {cert[0]}, code: {exit_code}')
@@ -221,7 +221,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
     @staticmethod
     def initialize_dictionary() -> Dict:
-        d = {'module_name': None, 'standard': None, 'status': None, 'date_sunset': None,
+        return {'module_name': None, 'standard': None, 'status': None, 'date_sunset': None,
              'date_validation': None, 'level': None, 'caveat': None, 'exceptions': None,
              'type': None, 'embodiment': None, 'tested_conf': None, 'description': None,
              'vendor': None, 'vendor_www': None, 'lab': None, 'lab_nvlap': None,
@@ -229,8 +229,6 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
              'mentioned_certs': {}, 'tables_done': False, 'security_policy_www': None, 'certificate_www': None,
              'hw_versions': None, 'fw_versions': None, 'sw_versions': None, 'product_url': None}
 
-        return d
-
     @staticmethod
     def parse_caveat(current_text: str) -> Dict[str, Dict[str, int]]:
         """
@@ -238,7 +236,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         :param current_text: text of "Caveat"
         :return: dictionary of all found algorithm IDs
         """
-        ids_found = {}
+        ids_found: Dict[str, Dict[str, int]] = {}
         r_key = r"(?P<word>\w+)?\s?(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)+(?P<id>\d+)"
         for m in re.finditer(r_key, current_text):
             if m.group('word') and m.group('word').lower() in {'rsa', 'shs', 'dsa', 'pkcs', 'aes'}:
@@ -462,7 +460,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                                    [] if not initialized else initialized.pdf_scan.algorithms,
                                    []  # connections
                                ),
-                               FIPSCertificate.FIPSHeuristics(None, {}, [], 0),
+                               FIPSCertificate.FIPSHeuristics(None, [], [], 0),
                                state
                                )
 
@@ -536,7 +534,11 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             else:
                 all_algorithms.add(''.join(filter(str.isdigit, x)))
         not_found = []
-        for alg_list in (a['Certificate'] for a in cert.web_scan.algorithms):
+        
+        if cert.web_scan.algorithms is None:
+            raise RuntimeError(f"Algorithms were not found for cert {cert.dgst} - this should not be happening.")
+        
+        for alg_list in [a['Certificate'] for a in cert.web_scan.algorithms]:
             for web_alg in alg_list:
                 if ''.join(filter(str.isdigit, web_alg)) not in all_algorithms:
                     not_found.append(web_alg)
@@ -552,9 +554,9 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
     @staticmethod
     def parse_cert_file_common(text_to_parse: str, whole_text_with_newlines: str,
-                               search_rules: Dict) -> Tuple[Optional[Dict[Pattern, Dict]], str]:
+                               search_rules: Dict) -> Tuple[Dict[Pattern, Dict], str]:
         # apply all rules
-        items_found_all = {}
+        items_found_all: Dict[Pattern, Dict] = {}
         for rule_group in search_rules.keys():
             if rule_group not in items_found_all:
                 items_found_all[rule_group] = {}
@@ -562,6 +564,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             items_found = items_found_all[rule_group]
 
             for rule in search_rules[rule_group]:
+                rule_and_sep: Union[Pattern, str]
                 if type(rule) != str:
                     rule_str = rule.pattern
                     rule_and_sep = re.compile(rule.pattern + REGEXEC_SEP)
@@ -620,7 +623,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         return items_found_all, whole_text_with_newlines
 
     @staticmethod
-    def parse_cert_file(text_to_parse: str) -> Tuple[Optional[Dict[Pattern, Dict]], str]:
+    def parse_cert_file(text_to_parse: str) -> Tuple[Dict[Pattern, Dict], str]:
         # apply all rules
         items_found_all: Dict = {}
 
@@ -694,7 +697,11 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         return True, cert, lst
 
     def _create_alg_set(self) -> Set:
-        result = set()
+        result: Set[str] = set()
+        
+        if self.web_scan.algorithms is None:
+            raise RuntimeError(f"Algorithms were not found for cert {self.dgst} - this should not be happening.")
+        
         for alg in self.web_scan.algorithms:
             result.update(cert for cert in alg['Certificate'])
         return result
@@ -752,14 +759,16 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             versions_for_extraction += f' {self.web_scan.fw_version}'
         self.heuristics.extracted_versions = helpers.compute_heuristics_version(versions_for_extraction)
 
-    # TODO: This function is probably safe to delete
+    # TODO: This function is probably safe to delete // I'll not type it then - older API probably?
     def compute_heuristics_cpe_vendors(self, cpe_dataset: CPEDataset):
-        self.heuristics.cpe_candidate_vendors = cpe_dataset.get_candidate_list_of_vendors(self.web_scan.vendor)
+        if self.web_scan.vendor is None:
+            raise RuntimeError(f"Vendor for cert {self.dgst} not found - this should not be happening.")
+        self.heuristics.cpe_candidate_vendors = cpe_dataset.get_candidate_list_of_vendors(self.web_scan.vendor) # type: ignore
 
     def compute_heuristics_cpe_match(self, cpe_classifier: CPEClassifier):
         if not self.web_scan.module_name:
             self.heuristics.cpe_matches = None
         else:
-            self.heuristics.cpe_matches = cpe_classifier.predict_single_cert(self.web_scan.vendor,
+            self.heuristics.cpe_matches = cpe_classifier.predict_single_cert(self.web_scan.vendor, # type: ignore
                                                                              self.web_scan.module_name,
                                                                              self.heuristics.extracted_versions)
diff --git a/sec_certs/sample/protection_profile.py b/sec_certs/sample/protection_profile.py
index 7e662abf..bb7cc788 100644
--- a/sec_certs/sample/protection_profile.py
+++ b/sec_certs/sample/protection_profile.py
@@ -15,7 +15,7 @@ class ProtectionProfile(ComplexSerializableType):
     """
     Object for holding protection profiles.
     """
-    pp_name: str
+    pp_name: Optional[str]
     pp_link: Optional[str] = None
     pp_ids: Optional[FrozenSet[str]] = None
 
diff --git a/sec_certs/serialization/json.py b/sec_certs/serialization/json.py
index 8760f8c2..1cd83cbc 100644
--- a/sec_certs/serialization/json.py
+++ b/sec_certs/serialization/json.py
@@ -1,7 +1,7 @@
 import json
 from datetime import date
 from pathlib import Path
-from typing import Dict, List, Union, Optional
+from typing import Dict, List, Union, Optional, Callable
 import copy
 
 
@@ -26,10 +26,12 @@ class ComplexSerializableType:
             raise TypeError(f'Dict: {dct} on {cls.__mro__}') from e
 
     def to_json(self, output_path: Optional[Union[str, Path]] = None):
-        if not output_path:
-            if not hasattr(self, 'json_path'):
+        if output_path is None and not hasattr(self, 'json_path'):
                 raise ValueError(f'The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument.')
-            output_path = self.json_path
+        elif output_path is None and self.json_path is None:
+            raise ValueError(f'The object {self} of type {self.__class__} does not have json_path attribute but to_json() was called without an argument.')
+        elif output_path is None:
+            output_path = self.json_path  # type: ignore
         with Path(output_path).open('w') as handle:
             json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
 
@@ -42,7 +44,7 @@ class ComplexSerializableType:
 
 
 # Decorator for serialization
-def serialize(func: callable):
+def serialize(func: Callable):
     def inner_func(*args, **kwargs):
         if not args or not issubclass(type(args[0]), ComplexSerializableType):
             raise ValueError('@serialize decorator is to be used only on instance methods of ComplexSerializableType child classes.')