Merge pull request #30 from petrs/fips-for-pr

History, web/pdf/processed, common/fips keyword matching

8 files changed, 448 insertions, 220 deletions

diff --git a/fips_oop_demo.py b/fips_oop_demo.py
index b9aa9425..561549b6 100644
--- a/fips_oop_demo.py
+++ b/fips_oop_demo.py
@@ -1,13 +1,19 @@
-from sec_certs.dataset import FIPSDataset, FIPSAlgorithmDataset
 from pathlib import Path
 from datetime import datetime
 import logging
+import click
+from sec_certs.dataset import FIPSDataset, FIPSAlgorithmDataset
+from sec_certs.configuration import config
 
-
-def main():
+@click.command()
+@click.option('--config-file', help='Path to config file')
+def main(config_file):
     logging.basicConfig(level=logging.INFO)
     start = datetime.now()
 
+    # Load config
+    config.load(config_file)
+
     # Create empty dataset
     dset = FIPSDataset({}, Path('./fips_dataset'), 'sample_dataset', 'sample dataset description')
 
@@ -15,7 +21,7 @@ def main():
     # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing')
 
     # Load metadata for certificates from CSV and HTML sources
-    dset.get_certs_from_web(True)
+    dset.get_certs_from_web()
 
     logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.')
     # Dump dataset into JSON
diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py
index 63187300..d8749495 100644
--- a/sec_certs/cert_rules.py
+++ b/sec_certs/cert_rules.py
@@ -413,7 +413,13 @@ rules_fips_remove_algorithm_ids = [
     r"#\d+, ?#\d+",
     r"#?\d+ and #?\d+",
     r"label \(#\d+\)",
-    r"\(#\d\)"
+    r"[Ll]abel #\d+",
+    r"\(#\d\)",
+    r"IETF[25\s]*RFC[26\s]*#\d+", # #3425
+    r"Bendix Road North #760", # #3325
+    r"5080 Spectrum Drive, #1000E",
+    r"Document # 540-105000-A1",
+    r"Certificate #2287-1 from EMCE Engineering", # ???
 ]
 
 rules_fips_cert = [
@@ -421,10 +427,10 @@ rules_fips_cert = [
     #     r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{3})",
     #     r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{2})",
     #     r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{1})
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{4})",
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{3})",
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2})",
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1})"
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{4}[^\d])",
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{3}[^\d])",
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2}[^\d])",
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1}[^\d])"
 ]
 
 #  rule still too "general"
@@ -498,8 +504,7 @@ fips_rules['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids
 fips_rules['rules_security_level'] = rules_fips_security_level
 fips_rules['rules_cert_id'] = rules_fips_cert
 fips_common_rules = copy.deepcopy(common_rules)  # make separate copy not to process cc rules by fips's re.compile
-fips_rules.update(fips_common_rules)
 
 for rule in fips_rules:
     for current_rule in range(len(fips_rules[rule])):
-        fips_rules[rule][current_rule] = re.compile(fips_rules[rule][current_rule] + REGEXEC_SEP)
\ No newline at end of file
+        fips_rules[rule][current_rule] = re.compile(fips_rules[rule][current_rule])
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index acb1fc79..39c56a3d 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -11,7 +11,7 @@ from dateutil import parser
 
 from abc import ABC, abstractmethod
 from bs4 import Tag, BeautifulSoup, NavigableString
-from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type, Tuple
+from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type, Tuple, Pattern
 
 from tabula import read_pdf
 
@@ -19,9 +19,9 @@ from sec_certs import helpers, extract_certificates, dataset
 from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
 import sec_certs.constants as constants
 from sec_certs.extract_certificates import load_cert_file, normalize_match_string, save_modified_cert_file, REGEXEC_SEP, \
-    LINE_SEPARATOR
-from sec_certs.cert_rules import fips_rules
-
+    LINE_SEPARATOR, APPEND_DETAILED_MATCH_MATCHES
+from sec_certs.cert_rules import fips_rules, fips_common_rules
+from sec_certs.configuration import config
 
 logger = logging.getLogger(__name__)
 
@@ -73,7 +73,8 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
         @classmethod
         def from_dict(cls, dct: Dict):
-            return cls(Path(dct['sp_path']), Path(dct['html_path']), Path(dct['fragment_path']))
+            return cls(Path(dct['sp_path']), Path(dct['html_path']), Path(dct['fragment_path']), dct['tables_done'],
+                       dct['file_status'], dct['txt_state'])
 
         def to_dict(self):
             return self.__dict__
@@ -81,6 +82,9 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         sp_path: Path
         html_path: Path
         fragment_path: Path
+        tables_done: bool
+        file_status: Optional[bool]
+        txt_state: bool
 
     @dataclass(eq=True, frozen=True)
     class Algorithm(ComplexSerializableType):
@@ -110,9 +114,109 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             return cls(dct['cert_id'], dct['vendor'], dct['implementation'], dct['type'],
                        dct['date'])
 
+    @dataclass(eq=True)
+    class WebScan(ComplexSerializableType):
+        module_name: Optional[str]
+        standard: Optional[str]
+        status: Optional[str]
+        date_sunset: Optional[Union[str, datetime]]
+        date_validation: Optional[List[Union[str, datetime]]]
+        level: Optional[str]
+        caveat: Optional[str]
+        exceptions: Optional[List[str]]
+        module_type: Optional[str]
+        embodiment: Optional[str]
+        algorithms: Optional[List[Dict[str, str]]]
+        tested_conf: Optional[List[str]]
+        description: Optional[str]
+        mentioned_certs: Optional[List[str]]
+        vendor: Optional[str]
+        vendor_www: Optional[str]
+        lab: Optional[str]
+        lab_nvlap: Optional[str]
+        historical_reason: Optional[str]
+        security_policy_www: Optional[str]
+        certificate_www: Optional[str]
+        hw_version: Optional[str]
+        fw_version: Optional[str]
+        revoked_reason: Optional[str]
+        revoked_link: Optional[str]
+        sw_versions: Optional[str]
+        product_url: Optional[str]
+
+        def __post_init__(self):
+            self.date_validation = [parser.parse(x) for x in self.date_validation] if self.date_validation else None
+            self.date_sunset = parser.parse(self.date_sunset) if self.date_sunset else None
+
+        @property
+        def dgst(self):
+            # certs in dataset are in format { id: [FIPSAlgorithm] }, there is only one type of algorithm
+            # for each id
+            return helpers.get_first_16_bytes_sha256(self.product_url + self.vendor_www)
+
+        def __repr__(self):
+            return self.module_name + ' created by ' + self.vendor
+
+        def __str__(self):
+            return str(self.module_name + ' created by ' + self.vendor)
+
+        def to_dict(self):
+            return copy.deepcopy(self.__dict__)
+
+        @classmethod
+        def from_dict(cls, dct: dict) -> 'FIPSCertificate.WebScan':
+            return cls(*tuple(dct.values()))
+
+    @dataclass(eq=True)
+    class PdfScan(ComplexSerializableType):
+        cert_id: int
+        keywords: Dict
+        algorithms: List
+
+        @property
+        def dgst(self):
+            # certs in dataset are in format { id: [FIPSAlgorithm] }, there is only one type of algorithm
+            # for each id
+            return helpers.get_first_16_bytes_sha256(str(self.keywords))
+
+        def __repr__(self):
+            return self.cert_id
+
+        def __str__(self):
+            return str(self.cert_id)
+
+        def to_dict(self):
+            return copy.deepcopy(self.__dict__)
+
+        @classmethod
+        def from_dict(cls, dct: dict) -> 'FIPSCertificate.PdfScan':
+            return cls(*tuple(dct.values()))
+
+    @dataclass(eq=True)
+    class Processed(ComplexSerializableType):
+        keywords: Optional[Dict]
+        algorithms: Dict
+        connections: List
+
+        @property
+        def dgst(self):
+            # certs in dataset are in format { id: [FIPSAlgorithm] }, there is only one type of algorithm
+            # for each id
+            return helpers.get_first_16_bytes_sha256(str(self.keywords))
+
+        def to_dict(self):
+            return copy.deepcopy(self.__dict__)
+
+        @classmethod
+        def from_dict(cls, dct: dict) -> 'FIPSCertificate.Processed':
+            return cls(*tuple(dct.values()))
+
     def __str__(self) -> str:
         return str(self.cert_id)
 
+    def to_dict(self) -> Dict:
+        return self.__dict__
+
     @property
     def dgst(self) -> str:
         return self.cert_id
@@ -124,80 +228,16 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             logger.error(f'Failed to download security policy from {cert[0]}, code: {exit_code}')
 
     def __init__(self, cert_id: str,
-                 module_name: Optional[str],
-                 standard: Optional[str],
-                 status: Optional[str],
-                 date_sunset: Optional[List[date]],
-                 date_validation: Optional[List[date]],
-                 level: Optional[str],
-                 caveat: Optional[str],
-                 exceptions: Optional[List[str]],
-                 module_type: Optional[str],
-                 embodiment: Optional[str],
-                 algorithms: Optional[List[Dict[str, str]]],
-                 tested_conf: Optional[List[str]],
-                 description: Optional[str],
-                 mentioned_certs: Optional[List[str]],
-                 vendor: Optional[str],
-                 vendor_www: Optional[str],
-                 lab: Optional[str],
-                 lab_nvlap: Optional[str],
-                 historical_reason: Optional[str],
-                 security_policy_www: Optional[str],
-                 certificate_www: Optional[str],
-                 hw_version: Optional[str],
-                 fw_version: Optional[str],
-                 tables: bool,
-                 file_status: Optional[bool],
-                 connections: List,
-                 state: State,
-                 txt_state: bool,
-                 keywords: Dict,
-                 revoked_reason: Optional[str],
-                 revoked_link: Optional[str],
-                 sw_versions: Optional[str],
-                 product_url: Optional[str]):
+                 web_scan: 'FIPSCertificate.WebScan',
+                 pdf_scan: 'FIPSCertificate.PdfScan',
+                 processed: 'FIPSCertificate.Processed',
+                 state: State):
         super().__init__()
         self.cert_id = cert_id
-
-        self.module_name = module_name
-        self.standard = standard
-        self.status = status
-        self.date_sunset = date_sunset
-        self.date_validation = date_validation
-        self.level = level
-        self.caveat = caveat
-        self.exceptions = exceptions
-        self.type = module_type
-        self.embodiment = embodiment
-
-        self.algorithms = algorithms
-        self.tested_conf = tested_conf
-        self.description = description
-        self.mentioned_certs = mentioned_certs
-        self.vendor = vendor
-        self.vendor_www = vendor_www
-        self.lab = lab
-        self.lab_nvlap = lab_nvlap
-
-        self.historical_reason = historical_reason
-
-        self.security_policy_www = security_policy_www
-        self.certificate_www = certificate_www
-        self.hw_versions = hw_version
-        self.fw_versions = fw_version
-
-        self.tables_done = tables
-        self.file_status = file_status
-        self.connections = connections
+        self.web_scan = web_scan
+        self.pdf_scan = pdf_scan
+        self.processed = processed
         self.state = state
-        self.txt_state = txt_state
-        self.keywords = keywords
-
-        self.revoked_reason = revoked_reason
-        self.revoked_link = revoked_link
-        self.sw_versions = sw_versions
-        self.product_url = product_url
 
     @staticmethod
     def download_html_page(cert: Tuple[str, Path]) -> None:
@@ -272,7 +312,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         for tr in trs:
             tds = tr.find_all('td')
             found_items.append(
-                {'Name': tds[0].text, 'Certificate': FIPSCertificate.extract_algorithm_certificates(tds[1].text)[0]['Certificate']})
+                {'Name': tds[0].text,
+                 'Certificate': FIPSCertificate.extract_algorithm_certificates(tds[1].text)[0]['Certificate'],
+                 'Links': [str(x) for x in tds[1].find_all('a')],
+                 'Raw': str(tr)})
 
         return found_items
 
@@ -284,10 +327,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
         if title in pairs:
             if 'date_validation' == pairs[title]:
-                html_items_found[pairs[title]] = [parser.parse(x) for x in content.split(';')]
-
-            elif 'date_sunset' == pairs[title]:
-                html_items_found[pairs[title]] = parser.parse(content)
+                html_items_found[pairs[title]] = [x for x in content.split(';')]
 
             elif 'caveat' in pairs[title]:
                 html_items_found[pairs[title]] = content
@@ -303,7 +343,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                 if 'Description' in title:
                     html_items_found['description'] = content
 
-            elif 'tested_conf' in pairs[title]:
+            elif 'tested_conf' in pairs[title] or 'exceptions' in pairs[title]:
                 html_items_found[pairs[title]] = [x.text for x in
                                                   current_div.find('div', class_='col-md-9').find_all('li')]
             else:
@@ -383,9 +423,15 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             items_found['cert_id'] = file.stem
 
         else:
-            items_found = initialized.__dict__
+            items_found = initialized.web_scan.__dict__
+            items_found['cert_id'] = initialized.cert_id
             items_found['revoked_reason'] = None
             items_found['revoked_link'] = None
+            items_found['mentioned_certs'] = []
+            state.tables_done = initialized.state.tables_done
+            state.file_status = initialized.state.file_status
+            state.txt_state = initialized.state.txt_state
+            initialized.processed.connections = []
 
         text = extract_certificates.load_cert_html_file(file)
         soup = BeautifulSoup(text, 'html.parser')
@@ -410,11 +456,15 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                     for cert_id in alg['Certificate']:
                         not_defined.add(cert_id)
                     continue
+
                 for pair in range(i + 1, len(items_found['algorithms'])):
                     if 'Name' in items_found['algorithms'][pair] \
                             and alg['Name'] == items_found['algorithms'][pair]['Name']:
                         entry = {'Name': alg['Name'], 'Certificate':
-                            list(set([x for x in alg['Certificate']]) | set(items_found['algorithms'][pair]['Certificate']))}
+                            list(set([x for x in alg['Certificate']])
+                                 | set(items_found['algorithms'][pair]['Certificate'])),
+                                 'Raw': items_found['algorithms'][pair]['Raw'],
+                                 'Links': items_found['algorithms'][pair]['Links']}
                         if entry not in new_algs:
                             new_algs.append(entry)
             for entry in new_algs:
@@ -424,76 +474,175 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             else:
                 new_algs.append({'Name': 'Not Defined', 'Certificate': list(not_defined)})
 
+
+            new_algs = [x for x in new_algs if x != {'Certificate': []}]
+
             items_found['algorithms'] = new_algs
 
         FIPSCertificate.normalize(items_found)
 
         return FIPSCertificate(items_found['cert_id'],
-                               items_found['module_name'],
-                               items_found['standard'],
-                               items_found['status'],
-                               items_found['date_sunset'],
-                               items_found['date_validation'],
-                               items_found['level'],
-                               items_found['caveat'],
-                               items_found['exceptions'],
-                               items_found['type'],
-                               items_found['embodiment'],
-                               items_found['algorithms'],
-                               items_found['tested_conf'],
-                               items_found['description'],
-                               items_found['mentioned_certs'],
-                               items_found['vendor'],
-                               items_found['vendor_www'],
-                               items_found['lab'],
-                               items_found['nvlap_code'],
-                               items_found['historical_reason'],
-                               items_found['security_policy_www'],
-                               items_found['certificate_www'],
-                               items_found['hw_versions'],
-                               items_found['fw_versions'],
-                               False if not initialized else items_found['tables_done'],
-                               None,
-                               [],
-                               state,
-                               False if not initialized else items_found['txt_state'],
-                               None if not initialized else items_found['keywords'],
-                               items_found['revoked_reason'],
-                               items_found['revoked_link'],
-                               items_found['sw_versions'],
-                               items_found['product_url'])
+                               FIPSCertificate.WebScan(
+                                   items_found['module_name'] if 'module_name' in items_found else None,
+                                   items_found['standard'] if 'standard' in items_found else None,
+                                   items_found['status'] if 'status' in items_found else None,
+                                   items_found['date_sunset'] if 'date_sunset' in items_found else None,
+                                   items_found['date_validation'] if 'date_validation' in items_found else None,
+                                   items_found['level'] if 'level' in items_found else None,
+                                   items_found['caveat'] if 'caveat' in items_found else None,
+                                   items_found['exceptions'] if 'exceptions' in items_found else None,
+                                   items_found['type'] if 'type' in items_found else None,
+                                   items_found['embodiment'] if 'embodiment' in items_found else None,
+                                   items_found['algorithms'] if 'algorithms' in items_found else None,
+                                   items_found['tested_conf'] if 'tested_conf' in items_found else None,
+                                   items_found['description'] if 'description' in items_found else None,
+                                   items_found['mentioned_certs'] if 'mentioned_certs' in items_found else None,
+                                   items_found['vendor'] if 'vendor' in items_found else None,
+                                   items_found['vendor_www'] if 'vendor_www' in items_found else None,
+                                   items_found['lab'] if 'lab' in items_found else None,
+                                   items_found['nvlap_code'] if 'nvlap_code' in items_found else None,
+                                   items_found['historical_reason'] if 'historical_reason' in items_found else None,
+                                   items_found['security_policy_www'] if 'security_policy_www' in items_found else None,
+                                   items_found['certificate_www'] if 'certificate_www' in items_found else None,
+                                   items_found['hw_versions'] if 'hw_versions' in items_found else None,
+                                   items_found['fw_versions'] if 'fw_versions' in items_found else None,
+                                   items_found['revoked_reason']  if 'revoked_reason' in items_found else None,
+                                   items_found['revoked_link'] if 'revoked_link' in items_found else None,
+                                   items_found['sw_versions'] if 'sw_versions' in items_found else None,
+                                   items_found['product_url']) if 'product_url' in items_found else None,
+                               FIPSCertificate.PdfScan(
+                                   items_found['cert_id'],
+                                   {} if not initialized else initialized.pdf_scan.keywords,
+                                   [] if not initialized else initialized.pdf_scan.algorithms
+                               ),
+                               FIPSCertificate.Processed(None, {}, []) if not initialized else initialized.processed,
+                               state
+                               )
 
     @staticmethod
     def convert_pdf_file(tup: Tuple['FIPSCertificate', Path, Path]) -> 'FIPSCertificate':
         cert, pdf_path, txt_path = tup
-        if not cert.txt_state:
+        if not cert.state.txt_state:
             exit_code = helpers.convert_pdf_file(pdf_path, txt_path, ['-raw'])
             if exit_code != constants.RETURNCODE_OK:
                 logger.error(f'Cert dgst: {cert.dgst} failed to convert security policy pdf->txt')
-                cert.txt_state = False
+                cert.state.txt_state = False
             else:
-                cert.txt_state = True
+                cert.state.txt_state = True
         return cert
 
     @staticmethod
-    def parse_cert_file(cert: 'FIPSCertificate') -> Tuple[Optional[Dict], 'FIPSCertificate']:
-        if not cert.txt_state:
+    def find_keywords(cert: 'FIPSCertificate') -> Tuple[Optional[Dict], 'FIPSCertificate']:
+        if not cert.state.txt_state:
             return None, cert
 
-        _, whole_text_with_newlines, unicode_error = load_cert_file(cert.state.sp_path.with_suffix('.pdf.txt'), -1,
-                                                                    LINE_SEPARATOR)
+        text, text_with_newlines, unicode_error = load_cert_file(cert.state.sp_path.with_suffix('.pdf.txt'),
+                                                                 -1, LINE_SEPARATOR)
+
+        text_to_parse = text_with_newlines if config.use_text_with_newlines_during_parsing['value'] else text
+
+        items_found, fips_text = FIPSCertificate.parse_cert_file(FIPSCertificate.remove_platforms(text_to_parse),
+                                                                 cert.web_scan.algorithms)
+
+        save_modified_cert_file(cert.state.fragment_path.with_suffix('.fips.txt'), fips_text, unicode_error)
+
+        common_items_found, common_text = FIPSCertificate.parse_cert_file_common(text_to_parse, text_with_newlines,
+                                                                                 fips_common_rules)
+
+        save_modified_cert_file(cert.state.fragment_path.with_suffix('.common.txt'), common_text, unicode_error)
+        items_found.update(common_items_found)
 
+        return items_found, cert
+
+    @staticmethod
+    def remove_platforms(text_to_parse: str):
+        pat = re.compile(r"(?:modification|revision|change) history\n[\s\S]*?", re.IGNORECASE)
+        for match in pat.finditer(text_to_parse):
+            text_to_parse = text_to_parse.replace(
+                match.group(), 'x' * len(match.group()))
+        return text_to_parse
+
+    @staticmethod
+    def parse_cert_file_common(text_to_parse: str, whole_text_with_newlines: str,
+                               search_rules: Dict) -> Tuple[Optional[Dict], str]:
         # apply all rules
         items_found_all = {}
-        for rule_group in fips_rules.keys():
+        for rule_group in search_rules.keys():
             if rule_group not in items_found_all:
                 items_found_all[rule_group] = {}
 
             items_found = items_found_all[rule_group]
 
+            for rule in search_rules[rule_group]:
+                if type(rule) != str:
+                    rule_str = rule.pattern
+                    rule_and_sep = re.compile(rule.pattern + REGEXEC_SEP)
+                else:
+                    rule_str = rule
+                    rule_and_sep = rule + REGEXEC_SEP
+
+                for m in re.finditer(rule_and_sep, text_to_parse):
+                    # insert rule if at least one match for it was found
+                    if rule not in items_found:
+                        items_found[rule_str] = {}
+
+                    match = m.group()
+                    match = normalize_match_string(match)
+
+                    MAX_ALLOWED_MATCH_LENGTH = 300
+                    match_len = len(match)
+                    if match_len > MAX_ALLOWED_MATCH_LENGTH:
+                        print('WARNING: Excessive match with length of {} detected for rule {}'.format(match_len, rule))
+
+                    if match not in items_found[rule_str]:
+                        items_found[rule_str][match] = {}
+                        items_found[rule_str][match][constants.TAG_MATCH_COUNTER] = 0
+                        if extract_certificates.APPEND_DETAILED_MATCH_MATCHES:
+                            items_found[rule_str][match][constants.TAG_MATCH_MATCHES] = []
+                        # else:
+                        #     items_found[rule_str][match][TAG_MATCH_MATCHES] = ['List of matches positions disabled. Set APPEND_DETAILED_MATCH_MATCHES to True']
+
+                    items_found[rule_str][match][constants.TAG_MATCH_COUNTER] += 1
+                    match_span = m.span()
+                    # estimate line in original text file
+                    # line_number = get_line_number(lines, line_length_compensation, match_span[0])
+                    # start index, end index, line number
+                    # items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number])
+                    if extract_certificates.APPEND_DETAILED_MATCH_MATCHES:
+                        items_found[rule_str][match][constants.TAG_MATCH_MATCHES].append(
+                            [match_span[0], match_span[1]])
+
+        # highlight all found strings (by xxxxx) from the input text and store the rest
+        all_matches = []
+        for rule_group in items_found_all.keys():
+            items_found = items_found_all[rule_group]
+            for rule in items_found.keys():
+                for match in items_found[rule]:
+                    all_matches.append(match)
+
+            # if AES string is removed before AES-128, -128 would be left in text => sort by length first
+            # sort before replacement based on the length of match
+            all_matches.sort(key=len, reverse=True)
+            for match in all_matches:
+                whole_text_with_newlines = whole_text_with_newlines.replace(
+                    match, 'x' * len(match))
+
+        return items_found_all, whole_text_with_newlines
+
+    @staticmethod
+    def parse_cert_file(text_to_parse: str, algorithms: List[Dict]) \
+            -> Tuple[Optional[Dict], str]:
+        # apply all rules
+        items_found_all: Dict = {}
+        for rule_group in fips_rules.keys():
+            if rule_group not in items_found_all:
+                items_found_all[rule_group] = {}
+
+            items_found: Dict[str, Dict] = items_found_all[rule_group]
+
             for rule in fips_rules[rule_group]:
-                for m in rule.finditer(whole_text_with_newlines):
-                # for m in re.finditer(rule, whole_text_with_newlines):
+                for m in rule.finditer(text_to_parse):
+                    # for m in re.finditer(rule, whole_text_with_newlines):
                     # insert rule if at least one match for it was found
                     if rule.pattern not in items_found:
                         items_found[rule.pattern] = {}
@@ -504,7 +653,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                     if match == '':
                         continue
 
-                    certs = [x['Certificate'] for x in cert.algorithms]
+                    certs = [x['Certificate'] for x in algorithms]
 
                     match_cert_id = ''.join(filter(str.isdigit, match))
 
@@ -519,20 +668,19 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
                     items_found[rule.pattern][match][constants.TAG_MATCH_COUNTER] += 1
 
-                    whole_text_with_newlines = whole_text_with_newlines.replace(
+                    text_to_parse = text_to_parse.replace(
                         match, 'x' * len(match))
 
-        save_modified_cert_file(cert.state.fragment_path, whole_text_with_newlines, unicode_error)
-        return items_found_all, cert
+        return items_found_all, text_to_parse
 
     @staticmethod
     def analyze_tables(cert: 'FIPSCertificate') -> Tuple[bool, 'FIPSCertificate', List]:
         cert_file = cert.state.sp_path
         txt_file = cert_file.with_suffix('.pdf.txt')
-        with open(txt_file, 'r') as f:
+        with open(txt_file, 'r', encoding='utf-8') as f:
             tables = helpers.find_tables(f.read(), txt_file)
 
-        lst = []
+        lst: List = []
         if tables:
             try:
                 data = read_pdf(cert_file, pages=tables, silent=True)
@@ -558,28 +706,40 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         return True, cert, lst
 
     def remove_algorithms(self):
-        self.file_status = True
-        if not self.keywords:
+        self.state.file_status = True
+        if not self.pdf_scan.keywords:
             return
 
-        if self.mentioned_certs:
-            for item in self.mentioned_certs:
-                self.keywords['rules_cert_id'].update(item)
+        self.processed.keywords = copy.deepcopy(self.pdf_scan.keywords)
+        if self.web_scan.mentioned_certs:
+            for item in self.web_scan.mentioned_certs:
+                self.processed.keywords['rules_cert_id'].update(item)
 
-        for rule in self.keywords['rules_cert_id']:
+        for rule in self.processed.keywords['rules_cert_id']:
             to_pop = set()
             rr = re.compile(rule)
-            for cert in self.keywords['rules_cert_id'][rule]:
-                for alg in self.keywords['rules_fips_algorithms']:
-                    for found in self.keywords['rules_fips_algorithms'][alg]:
+            for cert in self.processed.keywords['rules_cert_id'][rule]:
+                for alg in self.processed.keywords['rules_fips_algorithms']:
+                    for found in self.processed.keywords['rules_fips_algorithms'][alg]:
                         if rr.search(found) \
                                 and rr.search(cert) \
                                 and rr.search(found).group('id') == rr.search(cert).group('id'):
                             to_pop.add(cert)
+
+                for alg_cert in self.processed.algorithms:
+                    for cert_no in alg_cert['Certificate']:
+                        if int(''.join(filter(str.isdigit, cert_no))) == int(''.join(filter(str.isdigit, cert))):
+                            to_pop.add(cert)
             for r in to_pop:
-                self.keywords['rules_cert_id'][rule].pop(r, None)
+                self.processed.keywords['rules_cert_id'][rule].pop(r, None)
 
-            self.keywords['rules_cert_id'][rule].pop(self.cert_id, None)
+            self.processed.keywords['rules_cert_id'][rule].pop(self.cert_id, None)
+
+    @staticmethod
+    def get_compare(vendor: str):
+        vendor_split = vendor.replace(',', '') \
+            .replace('-', ' ').replace('+', ' ').replace('®', '').split()
+        return vendor_split[0] if len(vendor_split) > 0 else vendor
 
 
 class CommonCriteriaCert(Certificate, ComplexSerializableType):
diff --git a/sec_certs/configuration.py b/sec_certs/configuration.py
new file mode 100644
index 00000000..131a02d4
--- /dev/null
+++ b/sec_certs/configuration.py
@@ -0,0 +1,11 @@
+import yaml
+from pathlib import Path
+
+class Configuration:
+    def load(self, path: Path):
+        with open(path, 'r') as file:
+            state = yaml.load(file, Loader=yaml.FullLoader)
+        for k, v in state.items():
+            setattr(self, k, v)
+
+config = Configuration()
\ No newline at end of file
diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index 6d4aade5..1923937c 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -41,4 +41,5 @@ TAG_PP_SPONSOR = 'pp_sponsor'
 TAG_PP_EDITOR = 'pp_editor'
 TAG_PP_REVIEWER = 'pp_reviewer'
 TAG_KEYWORDS = 'keywords'
-FIPS_NOT_AVAILABLE_CERT_SIZE = 10000
\ No newline at end of file
+FIPS_NOT_AVAILABLE_CERT_SIZE = 10000
+FIPS_ALG_URL = 'https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/validation-search?searchMode=validation&page='
\ No newline at end of file
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 11747908..b852e6bc 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -3,6 +3,7 @@ from datetime import datetime
 import locale
 import logging
 from typing import Dict, List, ClassVar, Collection, Union, Set, Tuple
+from itertools import groupby
 
 import json
 from abc import ABC, abstractmethod
@@ -21,6 +22,7 @@ import sec_certs.files as files
 
 from sec_certs.certificate import CommonCriteriaCert, Certificate, FIPSCertificate
 from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
+from sec_certs.configuration import config
 
 logger = logging.getLogger(__name__)
 
@@ -518,21 +520,14 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
     def extract_keywords(self, redo=False):
         self.fragments_dir.mkdir(parents=True, exist_ok=True)
-        if self.new_files > 0 or not (self.root_dir / 'fips_full_keywords.json').exists():
 
-            keywords = cert_processing.process_parallel(FIPSCertificate.parse_cert_file,
-                                                        [cert for cert in self.certs.values() if not cert.keywords or redo],
-                                                        constants.N_THREADS,
-                                                        use_threading=False)
-            for keyword, cert in keywords:
-                self.certs[cert.dgst].keywords = keyword
-        else:
-            self.keywords = json.loads(
-                open(self.root_dir / 'fips_full_keywords.json').read())
-
-    def dump_keywords(self):
-        with open(self.root_dir / "fips_full_keywords.json", 'w') as f:
-            f.write(json.dumps(self.keywords, indent=4, sort_keys=True))
+        keywords = cert_processing.process_parallel(FIPSCertificate.find_keywords,
+                                                    [cert for cert in self.certs.values() if
+                                                     not cert.pdf_scan.keywords or redo],
+                                                    constants.N_THREADS,
+                                                    use_threading=False)
+        for keyword, cert in keywords:
+            self.certs[cert.dgst].pdf_scan.keywords = keyword
 
     def download_all_pdfs(self):
         sp_paths, sp_urls = [], []
@@ -566,11 +561,20 @@ class FIPSDataset(Dataset, ComplexSerializableType):
     def download_all_algs(self):
         algs_paths, algs_urls = [], []
 
+        # get first page to find out how many pages there are
+        helpers.download_file(
+            constants.FIPS_ALG_URL + '1',
+            self.algs_dir / "page1.html")
+
+        with open(self.algs_dir / "page1.html", "r") as alg_file:
+            soup = BeautifulSoup(alg_file.read(), 'html.parser')
+            num_pages = soup.select('span[data-total-pages]')[0].attrs
+
         self.algs_dir.mkdir(exist_ok=True)
-        for i in range(1, 502):
+        for i in range(1, int(num_pages['data-total-pages'])):
             if not (self.algs_dir / f'page{i}.html').exists():
                 algs_urls.append(
-                    f'https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/validation-search?searchMode=validation&page={i}')
+                    constants.FIPS_ALG_URL + str(i))
                 algs_paths.append(self.algs_dir / f"page{i}.html")
 
         logging.info(f"downloading {len(algs_urls)} algs html files")
@@ -582,7 +586,8 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         logger.info('Converting FIPS certificate reports to .txt')
         tuples = [
             (cert, self.policies_dir / f'{cert.cert_id}.pdf', self.policies_dir / f'{cert.cert_id}.pdf.txt')
-            for cert in self.certs.values() if not cert.txt_state and (self.policies_dir / f'{cert.cert_id}.pdf').exists()
+            for cert in self.certs.values() if
+            not cert.state.txt_state and (self.policies_dir / f'{cert.cert_id}.pdf').exists()
         ]
         cert_processing.process_parallel(FIPSCertificate.convert_pdf_file, tuples, constants.N_THREADS)
 
@@ -594,7 +599,8 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
         def get_certificates_from_html(html_file: Path) -> None:
             logger.info(f'Getting certificate ids from {html_file}')
-            html = BeautifulSoup(open(html_file).read(), 'html.parser')
+            with open(html_file, 'r', encoding='utf-8') as handle:
+                html = BeautifulSoup(handle.read(), 'html.parser')
 
             table = [x for x in html.find(
                 id='searchResultsTable').tbody.contents if x != '\n']
@@ -624,7 +630,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         for f in html_files:
             get_certificates_from_html(self.web_dir / f)
 
-        logger.info('Downloading certficate html and security policies')
+        logger.info('Downloading certificate html and security policies')
         download_html_pages()
 
         logger.info(f"{self.new_files} needed to be downloaded")
@@ -635,7 +641,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                     self.web_dir / f'{cert_id}.html',
                     FIPSCertificate.State((self.policies_dir / cert_id).with_suffix('.pdf'),
                                           (self.web_dir / cert_id).with_suffix('.html'),
-                                          (self.fragments_dir / cert_id).with_suffix('.txt')))
+                                          (self.fragments_dir / cert_id).with_suffix('.txt'), False, None, False))
             return
 
         logger.info("Certs loaded from previous scanning")
@@ -647,7 +653,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                     self.web_dir / f'{cert_id}.html',
                     FIPSCertificate.State((self.policies_dir / cert_id).with_suffix('.pdf'),
                                           (self.web_dir / cert_id).with_suffix('.html'),
-                                          (self.fragments_dir / cert_id).with_suffix('.txt')), cert)
+                                          (self.fragments_dir / cert_id).with_suffix('.txt'), False, None, False), cert)
 
     def extract_certs_from_tables(self) -> List[Path]:
         """
@@ -656,15 +662,15 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         """
         result = cert_processing.process_parallel(FIPSCertificate.analyze_tables,
                                                   [cert for cert in self.certs.values() if
-                                                   not cert.tables_done and cert.txt_state],
+                                                   not cert.state.tables_done and cert.state.txt_state],
                                                   constants.N_THREADS // 4,  # tabula already processes by parallel, so
-                                                                            # it's counterproductive to use all threads
+                                                  # it's counterproductive to use all threads
                                                   use_threading=False)
 
-        not_decoded = list(map(lambda tup: tup[1].state.sp_path, filter(lambda tup: tup[0] is False, result)))
+        not_decoded = [cert.state.sp_path for done, cert, _ in result if done is False]
         for state, cert, algorithms in result:
-            self.certs[cert.dgst].tables_done = state
-            self.certs[cert.dgst].algorithms += algorithms
+            self.certs[cert.dgst].state.tables_done = state
+            self.certs[cert.dgst].pdf_scan.algorithms += algorithms
 
         return not_decoded
 
@@ -673,14 +679,17 @@ class FIPSDataset(Dataset, ComplexSerializableType):
             cert.remove_algorithms()
 
     def unify_algorithms(self):
+        certificate: FIPSCertificate
         for certificate in self.certs.values():
             new_algorithms = []
-            for algorithm in certificate.algorithms:
+            united_algorithms = [x for x in (certificate.web_scan.algorithms + certificate.pdf_scan.algorithms) if
+                                 x != {'Certificate': []}]
+            for algorithm in united_algorithms:
                 if isinstance(algorithm, dict):
                     new_algorithms.append(algorithm)
                 else:
                     new_algorithms.append({'Certificate': [algorithm]})
-            certificate.algorithms = new_algorithms
+            certificate.processed.algorithms = new_algorithms
 
     def validate_results(self):
         """
@@ -688,22 +697,26 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         """
 
         def validate_id(processed_cert: FIPSCertificate, cert_candidate: str) -> bool:
+
             # returns True if candidates should _not_ be matched
             def compare_certs(current_certificate: 'FIPSCertificate', other_id: str):
-                cert_first = current_certificate.date_validation[0].year
-                cert_last = current_certificate.date_validation[-1].year
-                conn_first = self.certs[other_id].date_validation[0].year
-                conn_last = self.certs[other_id].date_validation[-1].year
+                cert_first = current_certificate.web_scan.date_validation[0].year
+                cert_last = current_certificate.web_scan.date_validation[-1].year
+                conn_first = self.certs[other_id].web_scan.date_validation[0].year
+                conn_last = self.certs[other_id].web_scan.date_validation[-1].year
 
-                return cert_first - conn_first > 5 and cert_last - conn_last > 5
+                return cert_first - conn_first > config.year_difference_between_validations['value'] \
+                       and cert_last - conn_last > config.year_difference_between_validations['value']
 
             # "< 105" still needs to be used, because of some old certs being revalidated
-            if cert_candidate.isdecimal() and (int(cert_candidate) < 105 or compare_certs(processed_cert, cert_candidate)):
+            if cert_candidate.isdecimal() \
+                    and int(cert_candidate) < config.smallest_certificate_id_to_connect['value'] or \
+                    compare_certs(processed_cert, cert_candidate):
                 return False
             if cert_candidate not in self.algorithms.certs:
                 return True
 
-            for cert_alg in processed_cert.algorithms:
+            for cert_alg in processed_cert.processed.algorithms:
                 for certificate in cert_alg['Certificate']:
                     curr_id = ''.join(filter(str.isdigit, certificate))
                     if curr_id == cert_candidate:
@@ -711,21 +724,25 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
             algs = self.algorithms.certs[cert_candidate]
             for current_alg in algs:
-                if processed_cert.vendor[:3] in current_alg.vendor:
+                if FIPSCertificate.get_compare(processed_cert.web_scan.vendor) == FIPSCertificate.get_compare(
+                        current_alg.vendor):
                     return False
             return True
 
         broken_files = set()
+
+        current_cert: FIPSCertificate
+
         for current_cert in self.certs.values():
-            if not current_cert.txt_state:
+            if not current_cert.state.txt_state:
                 continue
-            for rule in current_cert.keywords['rules_cert_id']:
-                for cert in current_cert.keywords['rules_cert_id'][rule]:
+            for rule in current_cert.processed.keywords['rules_cert_id']:
+                for cert in current_cert.processed.keywords['rules_cert_id'][rule]:
                     cert_id = ''.join(filter(str.isdigit, cert))
 
                     if cert_id == '' or cert_id not in self.certs:
                         broken_files.add(current_cert.dgst)
-                        current_cert.file_status = False
+                        current_cert.state.file_status = False
                         break
 
         if broken_files:
@@ -735,16 +752,16 @@ class FIPSDataset(Dataset, ComplexSerializableType):
             logger.warning(f"Total non-analyzable files:{len(broken_files)}")
 
         for current_cert in self.certs.values():
-            current_cert.connections = []
-            if not current_cert.file_status or not current_cert.keywords:
+            current_cert.processed.connections = []
+            if not current_cert.state.file_status or not current_cert.processed.keywords:
                 continue
-            if current_cert.keywords['rules_cert_id'] == {}:
+            if current_cert.processed.keywords['rules_cert_id'] == {}:
                 continue
-            for rule in current_cert.keywords['rules_cert_id']:
-                for cert in current_cert.keywords['rules_cert_id'][rule]:
+            for rule in current_cert.processed.keywords['rules_cert_id']:
+                for cert in current_cert.processed.keywords['rules_cert_id'][rule]:
                     cert_id = ''.join(filter(str.isdigit, cert))
-                    if cert_id not in current_cert.connections and validate_id(current_cert, cert_id):
-                        current_cert.connections.append(cert_id)
+                    if cert_id not in current_cert.processed.connections and validate_id(current_cert, cert_id):
+                        current_cert.processed.connections.append(cert_id)
 
     def finalize_results(self):
         self.unify_algorithms()
@@ -766,44 +783,48 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         dot.attr('node', style='filled')
 
         def found_interesting_cert(current_key):
-            if self.certs[current_key].vendor == highlighted_vendor:
+            if self.certs[current_key].web_scan.vendor == highlighted_vendor:
                 dot.attr('node', color='red')
-                if self.certs[current_key].status == 'Revoked':
+                if self.certs[current_key].web_scan.status == 'Revoked':
                     dot.attr('node', color='grey32')
-                if self.certs[current_key].status == 'Historical':
+                if self.certs[current_key].web_scan.status == 'Historical':
                     dot.attr('node', color='gold3')
-            if self.certs[current_key].vendor == "SUSE, LLC":
+            if self.certs[current_key].web_scan.vendor == "SUSE, LLC":
                 dot.attr('node', color='lightblue')
 
         def color_check(current_key):
             dot.attr('node', color='lightgreen')
-            if self.certs[current_key].status == 'Revoked':
+            if self.certs[current_key].web_scan.status == 'Revoked':
                 dot.attr('node', color='lightgrey')
-            if self.certs[current_key].status == 'Historical':
+            if self.certs[current_key].web_scan.status == 'Historical':
                 dot.attr('node', color='gold')
             found_interesting_cert(current_key)
-            dot.node(current_key, label=current_key + '\n' + self.certs[current_key].vendor +
-                                        ('\n' + self.certs[current_key].module_name
-                                         if self.certs[current_key].module_name else ''))
+            dot.node(current_key,
+                     label= current_key +
+                     '&#10;' +
+                     self.certs[current_key].web_scan.vendor +
+                     '&#10;' +
+                     (self.certs[current_key].web_scan.module_name
+                      if self.certs[current_key].web_scan.module_name else ''))
 
         keys = 0
         edges = 0
 
         highlighted_vendor = 'Red Hat®, Inc.'
         for key in self.certs:
-            if key != 'Not found' and self.certs[key].file_status:
-                if self.certs[key].connections:
+            if key != 'Not found' and self.certs[key].state.file_status:
+                if self.certs[key].processed.connections:
                     color_check(key)
                     keys += 1
                 else:
                     single_dot.attr('node', color='lightblue')
                     found_interesting_cert(key)
-                    single_dot.node(key, label=key + '\n' + self.certs[key].vendor + (
-                        '\n' + self.certs[key].module_name if self.certs[key].module_name else ''))
+                    single_dot.node(key, label=key + '\r\n' + self.certs[key].web_scan.vendor + (
+                        '\r\n' + self.certs[key].web_scan.module_name if self.certs[key].web_scan.module_name else ''))
 
         for key in self.certs:
-            if key != 'Not found' and self.certs[key].file_status:
-                for conn in self.certs[key].connections:
+            if key != 'Not found' and self.certs[key].state.file_status:
+                for conn in self.certs[key].processed.connections:
                     color_check(conn)
                     dot.edge(key, conn)
                     edges += 1
@@ -840,6 +861,15 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         dset.root_dir = input_path.parent.absolute()
         return dset
 
+    def group_vendors(self) -> Dict:
+        vendors = {}
+        v = {x.vendor.lower() for x in self.certs.values()}
+        v = sorted(v, key=FIPSCertificate.get_compare)
+        for prefix, a in groupby(v, key=FIPSCertificate.get_compare):
+            vendors[prefix] = list(a)
+
+        return vendors
+
 
 class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
 
@@ -853,7 +883,9 @@ class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
             return cert_type.strip(), cert_id.strip()
 
         for f in files.search_files(self.root_dir):
-            html_soup = BeautifulSoup(open(f).read(), 'html.parser')
+            with open(f, 'r', encoding='utf-8') as handle:
+                html_soup = BeautifulSoup(handle.read(), 'html.parser')
+
             table = html_soup.find('table', class_='table table-condensed publications-table table-bordered')
             spans = table.find_all('span')
             for span in spans:
diff --git a/sec_certs/settings.yaml b/sec_certs/settings.yaml
new file mode 100644
index 00000000..a35a18de
--- /dev/null
+++ b/sec_certs/settings.yaml
@@ -0,0 +1,12 @@
+---
+smallest_certificate_id_to_connect:
+  description: During validation we don't connect certificates with number lower than
+    _this_ to connections
+  value: 60
+year_difference_between_validations:
+  description: During validation we don't connect certificates with validation dates
+    difference higher than _this_
+  value: 5
+use_text_with_newlines_during_parsing:
+  description: During keyword search, search in text with newlines
+  value: true
diff --git a/setup.py b/setup.py
index f50e45b3..5ca6bff4 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,8 @@ setup(
         "requests",
         "tqdm",
         "beautifulsoup4",
-        "pandas"
+        "pandas",
+        "pyyaml"
     ],
     extras_require={
         "dev": ["mypy", "flake8"],