Split helpers to thematic chunks.

Split this disorientating junkyard of unrelated functions.

13 files changed, 1060 insertions, 1023 deletions

diff --git a/cc_cli.py b/cc_cli.py
index 1c28fdab..3877967b 100755
--- a/cc_cli.py
+++ b/cc_cli.py
@@ -9,7 +9,7 @@ import click
 
 from sec_certs.config.configuration import config
 from sec_certs.dataset import CCDataset
-from sec_certs.helpers import warn_if_missing_poppler
+from sec_certs.utils.helpers import warn_if_missing_poppler
 
 logger = logging.getLogger(__name__)
 
diff --git a/fips_cli.py b/fips_cli.py
index 5c95b5ae..1390a680 100755
--- a/fips_cli.py
+++ b/fips_cli.py
@@ -9,7 +9,7 @@ import click
 
 from sec_certs.config.configuration import DEFAULT_CONFIG_PATH, config
 from sec_certs.dataset import FIPSDataset
-from sec_certs.helpers import warn_if_missing_graphviz, warn_if_missing_poppler
+from sec_certs.utils.helpers import warn_if_missing_graphviz, warn_if_missing_poppler
 
 logger = logging.getLogger(__name__)
 
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index 03a7c207..7ec6cc13 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -15,6 +15,7 @@ import numpy as np
 import pandas as pd
 from bs4 import BeautifulSoup, Tag
 
+import sec_certs.utils.sanitization
 from sec_certs.utils import helpers as helpers
 from sec_certs.utils import parallel_processing as cert_processing
 from sec_certs.config.configuration import config
@@ -468,7 +469,7 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType):
 
         profiles = {
             x.dgst: set(
-                [ProtectionProfile(pp_name=y) for y in helpers.sanitize_protection_profiles(x.protection_profiles)]
+                [ProtectionProfile(pp_name=y) for y in sec_certs.utils.sanitization.sanitize_protection_profiles(x.protection_profiles)]
             )
             for x in df_base.itertuples()
         }
diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py
index 14656b3a..91df0d7f 100644
--- a/sec_certs/dataset/fips_algorithm.py
+++ b/sec_certs/dataset/fips_algorithm.py
@@ -5,6 +5,7 @@ from typing import Dict, List, Union
 
 from bs4 import BeautifulSoup
 
+import sec_certs.utils.extract
 from sec_certs import constants as constants
 from sec_certs.utils import helpers as helpers
 from sec_certs.utils import parallel_processing as cert_processing
@@ -70,7 +71,7 @@ class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
             cert_id = alg_string[len(cert_type) :]
             return cert_type.strip(), cert_id.strip()
 
-        for f in helpers.search_files(self.root_dir):
+        for f in sec_certs.utils.extract.search_files(self.root_dir):
             if not f.endswith("html"):
                 continue
 
diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py
index 7791425b..27326022 100644
--- a/sec_certs/sample/common_criteria.py
+++ b/sec_certs/sample/common_criteria.py
@@ -14,6 +14,9 @@ import numpy as np
 import requests
 from bs4 import Tag
 
+import sec_certs.utils.extract
+import sec_certs.utils.pdf
+import sec_certs.utils.sanitization
 from sec_certs import constants as constants
 from sec_certs.utils import helpers
 from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, security_level_csv_scan
@@ -24,11 +27,11 @@ from sec_certs.serialization.json import ComplexSerializableType
 from sec_certs.serialization.pandas import PandasSerializableType
 
 HEADERS = {
-    "anssi": helpers.search_only_headers_anssi,
-    "bsi": helpers.search_only_headers_bsi,
-    "nscib": helpers.search_only_headers_nscib,
-    "niap": helpers.search_only_headers_niap,
-    "canada": helpers.search_only_headers_canada,
+    "anssi": sec_certs.utils.extract.search_only_headers_anssi,
+    "bsi": sec_certs.utils.extract.search_only_headers_bsi,
+    "nscib": sec_certs.utils.extract.search_only_headers_nscib,
+    "niap": sec_certs.utils.extract.search_only_headers_niap,
+    "canada": sec_certs.utils.extract.search_only_headers_canada,
 }
 
 
@@ -64,10 +67,10 @@ class CommonCriteriaCert(
         maintenance_st_link: Optional[str]
 
         def __post_init__(self):
-            super().__setattr__("maintenance_report_link", helpers.sanitize_link(self.maintenance_report_link))
-            super().__setattr__("maintenance_st_link", helpers.sanitize_link(self.maintenance_st_link))
-            super().__setattr__("maintenance_title", helpers.sanitize_string(self.maintenance_title))
-            super().__setattr__("maintenance_date", helpers.sanitize_date(self.maintenance_date))
+            super().__setattr__("maintenance_report_link", sec_certs.utils.sanitization.sanitize_link(self.maintenance_report_link))
+            super().__setattr__("maintenance_st_link", sec_certs.utils.sanitization.sanitize_link(self.maintenance_st_link))
+            super().__setattr__("maintenance_title", sec_certs.utils.sanitization.sanitize_string(self.maintenance_title))
+            super().__setattr__("maintenance_date", sec_certs.utils.sanitization.sanitize_date(self.maintenance_date))
 
         @classmethod
         def from_dict(cls, dct: Dict) -> CommonCriteriaCert.MaintenanceReport:
@@ -373,20 +376,20 @@ class CommonCriteriaCert(
 
         self.status = status
         self.category = category
-        self.name = helpers.sanitize_string(name)
+        self.name = sec_certs.utils.sanitization.sanitize_string(name)
 
         self.manufacturer = None
         if manufacturer:
-            self.manufacturer = helpers.sanitize_string(manufacturer)
+            self.manufacturer = sec_certs.utils.sanitization.sanitize_string(manufacturer)
 
         self.scheme = scheme
-        self.security_level = helpers.sanitize_security_levels(security_level)
-        self.not_valid_before = helpers.sanitize_date(not_valid_before)
-        self.not_valid_after = helpers.sanitize_date(not_valid_after)
-        self.report_link = helpers.sanitize_link(report_link)
-        self.st_link = helpers.sanitize_link(st_link)
-        self.cert_link = helpers.sanitize_link(cert_link)
-        self.manufacturer_web = helpers.sanitize_link(manufacturer_web)
+        self.security_level = sec_certs.utils.sanitization.sanitize_security_levels(security_level)
+        self.not_valid_before = sec_certs.utils.sanitization.sanitize_date(not_valid_before)
+        self.not_valid_after = sec_certs.utils.sanitization.sanitize_date(not_valid_after)
+        self.report_link = sec_certs.utils.sanitization.sanitize_link(report_link)
+        self.st_link = sec_certs.utils.sanitization.sanitize_link(st_link)
+        self.cert_link = sec_certs.utils.sanitization.sanitize_link(cert_link)
+        self.manufacturer_web = sec_certs.utils.sanitization.sanitize_link(manufacturer_web)
         self.protection_profiles = protection_profiles
         self.maintenance_updates = maintenance_updates
         self.state = self.InternalState() if not state else state
@@ -728,7 +731,7 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: cert to download the pdf report for
         :return CommonCriteriaCert: the modified certificate with updated state
         """
-        exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path)
+        exit_code = sec_certs.utils.pdf.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path)
         if exit_code != constants.RETURNCODE_OK:
             error_msg = "failed to convert report pdf->txt"
             logger.error(f"Cert dgst: {cert.dgst}" + error_msg)
@@ -746,7 +749,7 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: cert to download the pdf security target for
         :return CommonCriteriaCert: the modified certificate with updated state
         """
-        exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path)
+        exit_code = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path)
         if exit_code != constants.RETURNCODE_OK:
             error_msg = "failed to convert security target pdf->txt"
             logger.error(f"Cert dgst: {cert.dgst}" + error_msg)
@@ -764,7 +767,7 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: cert to extract the metadata for.
         :return CommonCriteriaCert: the modified certificate with updated state
         """
-        response, cert.pdf_data.st_metadata = helpers.extract_pdf_metadata(cert.state.st_pdf_path)
+        response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path)
         if response != constants.RETURNCODE_OK:
             cert.state.st_extract_ok = False
             cert.state.errors.append(response)
@@ -778,7 +781,7 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: cert to extract the metadata for.
         :return CommonCriteriaCert: the modified certificate with updated state
         """
-        response, cert.pdf_data.report_metadata = helpers.extract_pdf_metadata(cert.state.report_pdf_path)
+        response, cert.pdf_data.report_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.report_pdf_path)
         if response != constants.RETURNCODE_OK:
             cert.state.report_extract_ok = False
             cert.state.errors.append(response)
@@ -835,7 +838,7 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: certificate to extract the keywords for.
         :return CommonCriteriaCert: the modified certificate with extracted keywords.
         """
-        response, cert.pdf_data.report_keywords = helpers.extract_keywords(cert.state.report_txt_path)
+        response, cert.pdf_data.report_keywords = sec_certs.utils.extract.extract_keywords(cert.state.report_txt_path)
         if response != constants.RETURNCODE_OK:
             cert.state.report_extract_ok = False
         return cert
@@ -849,7 +852,7 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: certificate to extract the keywords for.
         :return CommonCriteriaCert: the modified certificate with extracted keywords.
         """
-        response, cert.pdf_data.st_keywords = helpers.extract_keywords(cert.state.st_txt_path)
+        response, cert.pdf_data.st_keywords = sec_certs.utils.extract.extract_keywords(cert.state.st_txt_path)
         if response != constants.RETURNCODE_OK:
             cert.state.st_extract_ok = False
             cert.state.errors.append(response)
diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py
index da06d954..0207e5ff 100644
--- a/sec_certs/sample/fips.py
+++ b/sec_certs/sample/fips.py
@@ -13,11 +13,15 @@ from dateutil import parser
 from tabula import read_pdf
 
 import sec_certs.constants as constants
+import sec_certs.utils.extract
 import sec_certs.utils.helpers as helpers
+import sec_certs.utils.pdf
+import sec_certs.utils.tables
 from sec_certs.cert_rules import fips_rules
 from sec_certs.config.configuration import config
 from sec_certs.constants import LINE_SEPARATOR
-from sec_certs.utils.helpers import fips_dgst, load_cert_file, normalize_match_string, save_modified_cert_file
+from sec_certs.utils.helpers import fips_dgst
+from sec_certs.utils.extract import save_modified_cert_file, normalize_match_string, load_cert_file
 from sec_certs.sample.certificate import Certificate, Heuristics, References, logger
 from sec_certs.sample.cpe import CPE
 from sec_certs.serialization.json import ComplexSerializableType
@@ -542,7 +546,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
             items_found = FIPSCertificate._initialize_dictionary()
             items_found["cert_id"] = int(file.stem)
 
-        text = helpers.load_cert_html_file(str(file))
+        text = sec_certs.utils.extract.load_cert_html_file(str(file))
         soup = BeautifulSoup(text, "html.parser")
         for div in soup.find_all("div", class_="row padrow"):
             FIPSCertificate._parse_html_main(div, items_found, pairs)
@@ -612,7 +616,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
         """
         cert, pdf_path, txt_path = tup
         if not cert.state.txt_state:
-            exit_code = helpers.convert_pdf_file(pdf_path, txt_path)
+            exit_code = sec_certs.utils.pdf.convert_pdf_file(pdf_path, txt_path)
             if exit_code != constants.RETURNCODE_OK:
                 logger.error(f"Cert dgst: {cert.cert_id} failed to convert security policy pdf->txt")
                 cert.state.txt_state = False
@@ -811,7 +815,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
         cert_file = cert.state.sp_path
         txt_file = cert_file.with_suffix(".pdf.txt")
         with open(txt_file, "r", encoding="utf-8") as f:
-            tables = helpers.find_tables(f.read(), txt_file)
+            tables = sec_certs.utils.tables.find_tables(f.read(), txt_file)
         all_pages = precision and cert.heuristics.unmatched_algs > config.cert_threshold  # bool value
 
         lst: List = []
@@ -821,7 +825,7 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
             except Exception as e:
                 try:
                     logger.error(e)
-                    helpers.repair_pdf(cert_file)
+                    sec_certs.utils.pdf.repair_pdf(cert_file)
                     data = read_pdf(cert_file, pages="all" if all_pages else tables, silent=True)
 
                 except Exception as ex:
diff --git a/sec_certs/sample/protection_profile.py b/sec_certs/sample/protection_profile.py
index 136d545e..55fca1c0 100644
--- a/sec_certs/sample/protection_profile.py
+++ b/sec_certs/sample/protection_profile.py
@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from typing import Any, Dict, FrozenSet, Optional
 
 import sec_certs.utils.helpers as helpers
+import sec_certs.utils.sanitization
 from sec_certs.serialization.json import ComplexSerializableType
 
 logger = logging.getLogger(__name__)
@@ -20,8 +21,8 @@ class ProtectionProfile(ComplexSerializableType):
     pp_ids: Optional[FrozenSet[str]] = None
 
     def __post_init__(self):
-        super().__setattr__("pp_name", helpers.sanitize_string(self.pp_name))
-        super().__setattr__("pp_link", helpers.sanitize_link(self.pp_link))
+        super().__setattr__("pp_name", sec_certs.utils.sanitization.sanitize_string(self.pp_name))
+        super().__setattr__("pp_link", sec_certs.utils.sanitization.sanitize_link(self.pp_link))
 
     @classmethod
     def from_dict(cls, dct: Dict[str, Any]) -> "ProtectionProfile":
@@ -31,8 +32,8 @@ class ProtectionProfile(ComplexSerializableType):
 
     @classmethod
     def from_old_api_dict(cls, dct: Dict[str, Any]) -> "ProtectionProfile":
-        pp_name = helpers.sanitize_string(dct["csv_scan"]["cc_pp_name"])
-        pp_link = helpers.sanitize_link(dct["csv_scan"]["link_pp_document"])
+        pp_name = sec_certs.utils.sanitization.sanitize_string(dct["csv_scan"]["cc_pp_name"])
+        pp_link = sec_certs.utils.sanitization.sanitize_link(dct["csv_scan"]["link_pp_document"])
         pp_ids = frozenset(dct["processed"]["cc_pp_csvid"]) if dct["processed"]["cc_pp_csvid"] else None
         return cls(pp_name, pp_link, pp_ids)
 
diff --git a/sec_certs/utils/extract.py b/sec_certs/utils/extract.py
new file mode 100644
index 00000000..e2e1c63a
--- /dev/null
+++ b/sec_certs/utils/extract.py
@@ -0,0 +1,739 @@
+import os
+import re
+import logging
+from enum import Enum
+from pathlib import Path
+from typing import Dict, Tuple, Optional, Iterator, Union, Hashable, Generator, Any
+
+from sec_certs import constants as constants
+from sec_certs.cert_rules import REGEXEC_SEP, cc_rules as cc_search_rules
+from sec_certs.constants import LINE_SEPARATOR, MAX_ALLOWED_MATCH_LENGTH, TAG_MATCH_COUNTER, \
+    APPEND_DETAILED_MATCH_MATCHES, TAG_MATCH_MATCHES, FILE_ERRORS_STRATEGY
+
+logger = logging.getLogger(__name__)
+
+
+def search_only_headers_anssi(filepath: Path):  # noqa: C901
+    # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!!
+    class HEADER_TYPE(Enum):
+        HEADER_FULL = 1
+        HEADER_MISSING_CERT_ITEM_VERSION = 2
+        HEADER_MISSING_PROTECTION_PROFILES = 3
+        HEADER_DUPLICITIES = 4
+
+    rules_certificate_preface = [
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\\(s\\)(.+)Evaluation facility(.+)Recognition arrangements",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification report reference(.+)Product name \\(reference / version\\)(.+)TOE name \\(reference / version\\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements",
+        ),
+        (
+            HEADER_TYPE.HEADER_FULL,
+            "Certification report reference(.+)TOE name(.+)Product's reference/ version(.+)TOE's reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements",
+        ),
+        # corrupted text (duplicities)
+        (
+            HEADER_TYPE.HEADER_DUPLICITIES,
+            "Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles",
+        ),
+        # rules without product version
+        (
+            HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
+            "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
+            "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        (
+            HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+        # rules without protection profile
+        (
+            HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES,
+            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
+        ),
+    ]
+
+    # statistics about rules success rate
+    num_rules_hits = {}
+    for rule in rules_certificate_preface:
+        num_rules_hits[rule[1]] = 0
+
+    items_found = {}  # type: ignore # noqa
+
+    try:
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath)
+
+        # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs
+        pos = whole_text.find("")
+        if pos != -1:
+            pos = whole_text.find("", pos)
+            if pos != -1:
+                whole_text = whole_text[pos:]
+
+        no_match_yet = True
+        other_rule_already_match = False
+        rule_index = -1
+        for rule in rules_certificate_preface:
+            rule_index += 1
+            rule_and_sep = rule[1] + REGEXEC_SEP
+
+            for m in re.finditer(rule_and_sep, whole_text):
+                if no_match_yet:
+                    items_found[constants.TAG_HEADER_MATCH_RULES] = []
+                    no_match_yet = False
+
+                # insert rule if at least one match for it was found
+                if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]:
+                    items_found[constants.TAG_HEADER_MATCH_RULES].append(rule[1])
+
+                if not other_rule_already_match:
+                    other_rule_already_match = True
+                else:
+                    logger.warning(f"WARNING: multiple rules are matching same certification document: {filepath}")
+
+                num_rules_hits[rule[1]] += 1  # add hit to this rule
+                match_groups = m.groups()
+                index_next_item = 0
+                items_found[constants.TAG_CERT_ID] = normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+
+                items_found[constants.TAG_CERT_ITEM] = normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+
+                if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION:
+                    items_found[constants.TAG_CERT_ITEM_VERSION] = ""
+                else:
+                    items_found[constants.TAG_CERT_ITEM_VERSION] = normalize_match_string(match_groups[index_next_item])
+                    index_next_item += 1
+
+                if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES:
+                    items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = ""
+                else:
+                    items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
+                        match_groups[index_next_item]
+                    )
+                    index_next_item += 1
+
+                items_found[constants.TAG_CC_VERSION] = normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+
+                items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+
+                items_found[constants.TAG_DEVELOPER] = normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+
+                items_found[constants.TAG_CERT_LAB] = normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+    except Exception as e:
+        relative_filepath = "/".join(str(filepath).split("/")[-4:])
+        error_msg = f"Failed to parse ANSSI frontpage headers from {relative_filepath}; {e}"
+        logger.error(error_msg)
+        return error_msg, None
+
+    # if True:
+    #     print('# hits for rule')
+    #     sorted_rules = sorted(num_rules_hits.items(),
+    #                           key=operator.itemgetter(1), reverse=True)
+    #     used_rules = []
+    #     for rule in sorted_rules:
+    #         print('{:4d} : {}'.format(rule[1], rule[0]))
+    #         if rule[1] > 0:
+    #             used_rules.append(rule[0])
+
+    return constants.RETURNCODE_OK, items_found
+
+
+def search_only_headers_bsi(filepath: Path):  # noqa: C901
+    # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!!
+    LINE_SEPARATOR_STRICT = " "
+    NUM_LINES_TO_INVESTIGATE = 15
+    rules_certificate_preface = [
+        "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)",
+        "(BSI-DSZ-CC-.+?) zu (.+?) der (.*)",
+    ]
+
+    items_found = {}  # type: ignore # noqa
+    no_match_yet = True
+
+    try:
+        # Process front page with info: cert_id, certified_item and developer
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
+        )
+
+        for rule in rules_certificate_preface:
+            rule_and_sep = rule + REGEXEC_SEP
+
+            for m in re.finditer(rule_and_sep, whole_text):
+                if no_match_yet:
+                    items_found[constants.TAG_HEADER_MATCH_RULES] = []
+                    no_match_yet = False
+
+                # insert rule if at least one match for it was found
+                if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]:
+                    items_found[constants.TAG_HEADER_MATCH_RULES].append(rule)
+
+                match_groups = m.groups()
+                cert_id = match_groups[0]
+                certified_item = match_groups[1]
+                developer = match_groups[2]
+
+                FROM_KEYWORD_LIST = [" from ", " der "]
+                for from_keyword in FROM_KEYWORD_LIST:
+                    from_keyword_len = len(from_keyword)
+                    if certified_item.find(from_keyword) != -1:
+                        logger.warning(
+                            f"string {from_keyword} detected in certified item - shall not be here, fixing..."
+                        )
+                        certified_item_first = certified_item[: certified_item.find(from_keyword)]
+                        developer = certified_item[certified_item.find(from_keyword) + from_keyword_len :]
+                        certified_item = certified_item_first
+                        continue
+
+                end_pos = developer.find("\f-")
+                if end_pos == -1:
+                    end_pos = developer.find("\fBSI")
+                if end_pos == -1:
+                    end_pos = developer.find("Bundesamt")
+                if end_pos != -1:
+                    developer = developer[:end_pos]
+
+                items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
+                items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item)
+                items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer)
+                items_found[constants.TAG_CERT_LAB] = "BSI"
+
+        # Process page with more detailed sample info
+        # PP Conformance, Functionality, Assurance
+        rules_certificate_third = ["PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified"]
+
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath)
+
+        for rule in rules_certificate_third:
+            rule_and_sep = rule + REGEXEC_SEP
+
+            for m in re.finditer(rule_and_sep, whole_text):
+                # check if previous rules had at least one match
+                if constants.TAG_CERT_ID not in items_found.keys():
+                    logger.error("ERROR: front page not found for file: {}".format(filepath))
+
+                match_groups = m.groups()
+                ref_protection_profiles = match_groups[0]
+                cc_version = match_groups[1]
+                cc_security_level = match_groups[2]
+
+                items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
+                    ref_protection_profiles
+                )
+                items_found[constants.TAG_CC_VERSION] = normalize_match_string(cc_version)
+                items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(cc_security_level)
+
+        # print('\n*** Certificates without detected preface:')
+        # for file_name in files_without_match:
+        #     print('No hits for {}'.format(file_name))
+        # print('Total no hits files: {}'.format(len(files_without_match)))
+        # print('\n**********************************')
+    except Exception as e:
+        relative_filepath = "/".join(str(filepath).split("/")[-4:])
+        error_msg = f"Failed to parse BSI headers from frontpage: {relative_filepath}; {e}"
+        logger.error(error_msg)
+        return error_msg, None
+
+    return constants.RETURNCODE_OK, items_found
+
+
+def search_only_headers_nscib(filepath: Path):  # noqa: C901
+    # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!!
+    LINE_SEPARATOR_STRICT = " "
+    NUM_LINES_TO_INVESTIGATE = 60
+    items_found: Dict[str, str] = {}
+
+    try:
+        # Process front page with info: cert_id, certified_item and developer
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
+        )
+
+        certified_item = ""
+        developer = ""
+        cert_lab = ""
+        cert_id = ""
+
+        lines = whole_text_with_newlines.splitlines()
+        no_match_yet = True
+        item_offset = -1
+
+        for line_index in range(0, len(lines)):
+            line = lines[line_index]
+
+            if "Certification Report" in line:
+                item_offset = line_index + 1
+            if "Assurance Continuity Maintenance Report" in line:
+                item_offset = line_index + 1
+
+            SPONSORDEVELOPER_STR = "Sponsor and developer:"
+
+            if SPONSORDEVELOPER_STR in line:
+                if no_match_yet:
+                    items_found = {}
+                    no_match_yet = False
+
+                # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report'
+                certified_item = ""
+                for name_index in range(item_offset, line_index):
+                    certified_item += lines[name_index] + " "
+                developer = line[line.find(SPONSORDEVELOPER_STR) + len(SPONSORDEVELOPER_STR) :]
+
+            SPONSOR_STR = "Sponsor:"
+
+            if SPONSOR_STR in line:
+                if no_match_yet:
+                    items_found = {}
+                    no_match_yet = False
+
+                # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report'
+                certified_item = ""
+                for name_index in range(item_offset, line_index):
+                    certified_item += lines[name_index] + " "
+
+            DEVELOPER_STR = "Developer:"
+            if DEVELOPER_STR in line:
+                developer = line[line.find(DEVELOPER_STR) + len(DEVELOPER_STR) :]
+
+            CERTLAB_STR = "Evaluation facility:"
+            if CERTLAB_STR in line:
+                cert_lab = line[line.find(CERTLAB_STR) + len(CERTLAB_STR) :]
+
+            REPORTNUM_STR = "Report number:"
+            if REPORTNUM_STR in line:
+                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
+
+        if not no_match_yet:
+            items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
+            items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item)
+            items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer)
+            items_found[constants.TAG_CERT_LAB] = cert_lab
+
+    except Exception as e:
+        error_msg = f"Failed to parse NSCIB headers from frontpage: {filepath}; {e}"
+        logger.error(error_msg)
+        return error_msg, None
+
+    return constants.RETURNCODE_OK, items_found
+
+
+def search_only_headers_niap(filepath: Path):
+    # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!!
+    LINE_SEPARATOR_STRICT = " "
+    NUM_LINES_TO_INVESTIGATE = 15
+    items_found: Dict[str, str] = {}
+
+    try:
+        # Process front page with info: cert_id, certified_item and developer
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
+        )
+
+        certified_item = ""
+        cert_id = ""
+
+        lines = whole_text_with_newlines.splitlines()
+        no_match_yet = True
+        item_offset = -1
+
+        for line_index in range(0, len(lines)):
+            line = lines[line_index]
+
+            if "Validation Report" in line:
+                item_offset = line_index + 1
+
+            REPORTNUM_STR = "Report Number:"
+            if REPORTNUM_STR in line:
+                if no_match_yet:
+                    items_found = {}
+                    no_match_yet = False
+
+                # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report'
+                certified_item = ""
+                for name_index in range(item_offset, line_index):
+                    certified_item += lines[name_index] + " "
+                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
+                break
+
+        if not no_match_yet:
+            items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
+            items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item)
+            items_found[constants.TAG_CERT_LAB] = "US NIAP"
+
+    except Exception as e:
+        error_msg = f"Failed to parse NIAP headers from frontpage: {filepath}; {e}"
+        logger.error(error_msg)
+        return error_msg, None
+
+    return constants.RETURNCODE_OK, items_found
+
+
+def search_only_headers_canada(filepath: Path):  # noqa: C901
+    # TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!!
+    LINE_SEPARATOR_STRICT = " "
+    NUM_LINES_TO_INVESTIGATE = 20
+    items_found: Dict[str, str] = {}
+    try:
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
+        )
+
+        cert_id = ""
+
+        lines = whole_text_with_newlines.splitlines()
+        no_match_yet = True
+        for line_index in range(0, len(lines)):
+            line = lines[line_index]
+            if "Government of Canada, Communications Security Establishment" in line:
+                REPORTNUM_STR1 = "Evaluation number:"
+                REPORTNUM_STR2 = "Document number:"
+                matched_number_str = ""
+                line_certid = lines[line_index + 1]
+                if line_certid.startswith(REPORTNUM_STR1):
+                    matched_number_str = REPORTNUM_STR1
+                if line_certid.startswith(REPORTNUM_STR2):
+                    matched_number_str = REPORTNUM_STR2
+                if matched_number_str != "":
+                    if no_match_yet:
+                        items_found = {}
+                        no_match_yet = False
+
+                    cert_id = line_certid[line_certid.find(matched_number_str) + len(matched_number_str) :]
+                    break
+
+            if (
+                "Government of Canada. This document is the property of the Government of Canada. It shall not be altered,"
+                in line
+            ):
+                REPORTNUM_STR = "Evaluation number:"
+                for offset in range(1, 20):
+                    line_certid = lines[line_index + offset]
+                    if "UNCLASSIFIED" in line_certid:
+                        if no_match_yet:
+                            items_found = {}
+                            no_match_yet = False
+                        line_certid = lines[line_index + offset - 4]
+                        cert_id = line_certid[line_certid.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
+                        break
+                if not no_match_yet:
+                    break
+
+            if (
+                "UNCLASSIFIED / NON CLASSIFIÉ" in line
+                and "COMMON CRITERIA CERTIFICATION REPORT" in lines[line_index + 2]
+            ):
+                line_certid = lines[line_index + 1]
+                if no_match_yet:
+                    items_found = {}
+                    no_match_yet = False
+                cert_id = line_certid
+                break
+
+        if not no_match_yet:
+            items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
+            items_found[constants.TAG_CERT_LAB] = "CANADA"
+
+    except Exception as e:
+        error_msg = f"Failed to parse Canada headers from frontpage: {filepath}; {e}"
+        logger.error(error_msg)
+        return error_msg, None
+
+    return constants.RETURNCODE_OK, items_found
+
+
+def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, int]]]]:
+    try:
+        result = parse_cert_file(filepath, cc_search_rules, -1, constants.LINE_SEPARATOR)
+
+        processed_result = {}
+        top_level_keys = list(result.keys())
+        for key in top_level_keys:
+            processed_result[key] = {key: val for key, val in gen_dict_extract(result[key])}
+
+    except Exception as e:
+        relative_filepath = "/".join(str(filepath).split("/")[-4:])
+        error_msg = f"Failed to parse keywords from: {relative_filepath}; {e}"
+        logger.error(error_msg)
+        return error_msg, None
+    return constants.RETURNCODE_OK, processed_result
+
+
+def search_files(folder: str) -> Iterator[str]:
+    for root, _, files in os.walk(folder):
+        yield from [os.path.join(root, x) for x in files]
+
+
+def save_modified_cert_file(target_file: Union[str, Path], modified_cert_file_text: str, is_unicode_text: bool) -> None:
+    if is_unicode_text:
+        write_file = Path(target_file).open("w", encoding="utf8", errors="replace")
+    else:
+        write_file = Path(target_file).open("w", errors="replace")
+
+    try:
+        write_file.write(modified_cert_file_text)
+    except UnicodeEncodeError:
+        print("UnicodeDecodeError while writing file fragments back")
+    finally:
+        write_file.close()
+
+
+def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR):  # noqa: C901
+    whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+        file_name, limit_max_lines, line_separator
+    )
+
+    items_found_all = {}
+    for rule_group, rules in search_rules.items():
+        if rule_group not in items_found_all:
+            items_found_all[rule_group] = {}
+
+        items_found = items_found_all[rule_group]
+
+        for rule in rules:
+            rule_str, rule_and_sep = rule
+
+            for m in re.finditer(rule_and_sep, whole_text):
+                if rule_str not in items_found:
+                    items_found[rule_str] = {}
+
+                match = m.group()
+                match = normalize_match_string(match)
+
+                match_len = len(match)
+                if match_len > MAX_ALLOWED_MATCH_LENGTH:
+                    logger.warning(f"Excessive match with length of {match_len} detected for rule {rule_str}")
+
+                if match not in items_found[rule_str]:
+                    items_found[rule_str][match] = {}
+                    items_found[rule_str][match][TAG_MATCH_COUNTER] = 0
+                    if APPEND_DETAILED_MATCH_MATCHES:
+                        items_found[rule_str][match][TAG_MATCH_MATCHES] = []
+                items_found[rule_str][match][TAG_MATCH_COUNTER] += 1
+                match_span = m.span()
+                if APPEND_DETAILED_MATCH_MATCHES:
+                    items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1]])
+
+    return items_found_all
+
+
+def normalize_match_string(match: str) -> str:
+    match = match.strip().rstrip('];.”":)(,').rstrip(os.sep).replace("  ", " ")
+    return "".join(filter(str.isprintable, match))
+
+
+def load_cert_file(
+    file_name: Union[str, Path], limit_max_lines: int = -1, line_separator: str = LINE_SEPARATOR
+) -> Tuple[str, str, bool]:
+    lines = []
+    was_unicode_decode_error = False
+    with Path(file_name).open("r", errors=FILE_ERRORS_STRATEGY) as f:
+        try:
+            lines = f.readlines()
+        except UnicodeDecodeError:
+            f.close()
+            was_unicode_decode_error = True
+            print("  WARNING: UnicodeDecodeError, opening as utf8")
+
+            with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
+                # coding failure, try line by line
+                line = " "
+                while line:
+                    try:
+                        line = f2.readline()
+                        lines.append(line)
+                    except UnicodeDecodeError:
+                        # ignore error
+                        continue
+
+    whole_text = ""
+    whole_text_with_newlines = ""
+    # we will estimate the line for searched matches
+    # => we need to known how much lines were modified (removal of eoln..)
+    # for removed newline and for any added separator
+    # line_length_compensation = 1 - len(LINE_SEPARATOR)
+    lines_included = 0
+    for line in lines:
+        if limit_max_lines != -1 and lines_included >= limit_max_lines:
+            break
+
+        whole_text_with_newlines += line
+        line = line.replace("\n", "")
+        whole_text += line
+        whole_text += line_separator
+        lines_included += 1
+
+    return whole_text, whole_text_with_newlines, was_unicode_decode_error
+
+
+def load_cert_html_file(file_name: str) -> str:
+    with open(file_name, "r", errors=FILE_ERRORS_STRATEGY) as f:
+        try:
+            whole_text = f.read()
+        except UnicodeDecodeError:
+            f.close()
+            with open(file_name, "r", encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
+                try:
+                    whole_text = f2.read()
+                except UnicodeDecodeError:
+                    print("### ERROR: failed to read file {}".format(file_name))
+    return whole_text
+
+
+def gen_dict_extract(dct: Dict, searched_key: Hashable = "count") -> Generator[Any, None, None]:
+    """
+    Function to flatten dictionary with some serious limitations. We only expect to use it temporarily on dictionary
+    produced by extract_keywords that contains many layers. On the deepest level in that dictionary, 'some_match': {'count': frequency}.
+    The output of the function will be list of tuples ('some_match': frequency)
+    :param searched_key: key to search, 'count'
+    :param dct: Dictionary to search
+    :return: List of tuples
+    """
+    for key, value in dct.items():
+        if key == searched_key:
+            yield value
+        if isinstance(value, dict):
+            for result in gen_dict_extract(value, searched_key):
+                if isinstance(result, tuple):
+                    yield result
+                else:
+                    yield key, result
\ No newline at end of file
diff --git a/sec_certs/utils/helpers.py b/sec_certs/utils/helpers.py
index 21451c97..aef5afb8 100644
--- a/sec_certs/utils/helpers.py
+++ b/sec_certs/utils/helpers.py
@@ -1,40 +1,21 @@
 import hashlib
-import html
 import logging
-import os
 import re
 import time
 from contextlib import nullcontext
-from datetime import date, datetime, timedelta, timezone
-from enum import Enum
-from functools import partial, reduce
+from datetime import datetime
+from functools import partial
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
-from typing import Any, Dict, Generator, Hashable, Iterator, List, Optional, Sequence, Set, Tuple, Union
+from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
 
-import matplotlib.pyplot as plt
 import numpy as np
-import pandas as pd
-import pdftotext
-import pikepdf
 import pkgconfig
 import requests
-from PyPDF2 import PdfFileReader
-from PyPDF2.generic import BooleanObject, FloatObject, IndirectObject, NumberObject
 from tqdm import tqdm as tqdm_original
 
 import sec_certs.constants as constants
-from sec_certs.cert_rules import REGEXEC_SEP
-from sec_certs.cert_rules import cc_rules as cc_search_rules
 from sec_certs.config.configuration import config
-from sec_certs.constants import (
-    APPEND_DETAILED_MATCH_MATCHES,
-    FILE_ERRORS_STRATEGY,
-    LINE_SEPARATOR,
-    MAX_ALLOWED_MATCH_LENGTH,
-    TAG_MATCH_COUNTER,
-    TAG_MATCH_MATCHES,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -116,221 +97,6 @@ def get_sha256_filepath(filepath: Union[str, Path]) -> str:
     return hash_sha256.hexdigest()
 
 
-def sanitize_link(record: Optional[str]) -> Optional[str]:
-    if not record:
-        return None
-    return record.replace(":443", "").replace(" ", "%20").replace("http://", "https://")
-
-
-def sanitize_date(record: Union[pd.Timestamp, date, np.datetime64]) -> Union[date, None]:
-    if pd.isnull(record):
-        return None
-    elif isinstance(record, pd.Timestamp):
-        return record.date()
-    elif isinstance(record, (date, type(None))):
-        return record
-    raise ValueError("Unsupported type given as input")
-
-
-def sanitize_string(record: str) -> str:
-    # There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I &#38&#x3b; J' that has to be unescaped twice
-    string = html.unescape(html.unescape(record)).replace("\n", "")
-    return " ".join(string.split())
-
-
-def sanitize_security_levels(record: Union[str, Set[str]]) -> Set[str]:
-    if isinstance(record, str):
-        record = set(record.split(","))
-    return record - {"Basic", "ND-PP", "PP\xa0Compliant", "None"}
-
-
-def sanitize_protection_profiles(record: str) -> list:
-    if not record:
-        return []
-    return record.split(",")
-
-
-def parse_list_of_tables(txt: str) -> Set[str]:
-    """
-    Parses list of tables from function find_tables(), finds ones that mention algorithms
-    :param txt: chunk of text
-    :return: set of all pages mentioning algorithm table
-    """
-    rr = re.compile(r"^.+?(?:[Ff]unction|[Aa]lgorithm|[Ss]ecurity [Ff]unctions?).+?(?P<page_num>\d+)$", re.MULTILINE)
-    pages = set()
-    for m in rr.finditer(txt):
-        pages.add(m.group("page_num"))
-    return pages
-
-
-def find_tables_iterative(file_text: str) -> List[int]:
-    current_page = 1
-    pages = set()
-    for line in file_text.split("\n"):
-        if "\f" in line:
-            current_page += 1
-        if line.startswith("Table ") or line.startswith("Exhibit"):
-            pages.add(current_page)
-            pages.add(current_page + 1)
-            if current_page > 2:
-                pages.add(current_page - 1)
-    if not pages:
-        logger.warning("No pages found")
-    for page in pages:
-        if page > current_page - 1:
-            return list(pages - {page})
-
-    return list(pages)
-
-
-def find_tables(txt: str, file_name: Path) -> Optional[Union[List[str], List[int]]]:
-    """
-    Function that tries to pages in security policy pdf files, where it's possible to find a table containing
-    algorithms
-    :param txt: file in .txt format (output of pdftotext)
-    :param file_name: name of the file
-    :return:    list of pages possibly containing a table
-                None if these cannot be found
-    """
-    # Look for "List of Tables", where we can find exactly tables with page num
-    tables_regex = re.compile(r"^(?:(?:[Tt]able\s|[Ll]ist\s)(?:[Oo]f\s))[Tt]ables[\s\S]+?\f", re.MULTILINE)
-    table = tables_regex.search(txt)
-    if table:
-        rb = parse_list_of_tables(table.group())
-        if rb:
-            return list(rb)
-        return None
-
-    # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer
-    logger.info(f"parsing tables in {file_name}")
-    table_page_indices = find_tables_iterative(txt)
-    return table_page_indices if table_page_indices else None
-
-
-def repair_pdf(file: Path) -> None:
-    """
-    Some pdfs can't be opened by PyPDF2 - opening them with pikepdf and then saving them fixes this issue.
-    By opening this file in a pdf reader, we can already extract number of pages
-    :param file: file name
-    :return: number of pages in pdf file
-    """
-    pdf = pikepdf.Pdf.open(file, allow_overwriting_input=True)
-    pdf.save(file)
-
-
-def convert_pdf_file(pdf_path: Path, txt_path: Path) -> str:
-    try:
-        with pdf_path.open("rb") as pdf_handle:
-            pdf = pdftotext.PDF(pdf_handle, "", True)  # No password, Raw=True
-            txt = "".join(pdf)
-    except Exception as e:
-        logger.error(f"Error when converting pdf->txt: {e}")
-        return constants.RETURNCODE_NOK
-
-    with txt_path.open("w", encoding="utf-8") as txt_handle:
-        txt_handle.write(txt)
-
-    return constants.RETURNCODE_OK
-
-
-def parse_pdf_date(dateval) -> Optional[datetime]:
-    """
-    Parse PDF metadata date format:
-
-    ```
-        parse_pdf_date(b"D:20110617082321-04'00'")
-    ```
-    into
-    ```
-        datetime.datetime(2011, 6, 17, 8, 23, 21, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=72000)))
-    ```
-    """
-    if dateval is None:
-        return None
-    clean = dateval.decode("utf-8").replace("D:", "")
-    tz = None
-    tzoff = None
-    if "+" in clean:
-        clean, tz = clean.split("+")
-        tzoff = 1
-    if "-" in clean:
-        clean, tz = clean.split("-")
-        tzoff = -1
-    elif "Z" in clean:
-        clean, tz = clean.split("Z")
-        tzoff = 1
-    try:
-        res_datetime = datetime.strptime(clean, "%Y%m%d%H%M%S")
-        if tz and tzoff:
-            tz_datetime = datetime.strptime(tz, "%H'%M'")
-            delta = tzoff * timedelta(hours=tz_datetime.hour, minutes=tz_datetime.minute)
-            res_tz = timezone(delta)
-            res_datetime = res_datetime.replace(tzinfo=res_tz)
-        return res_datetime
-    except ValueError:
-        return None
-
-
-def extract_pdf_metadata(filepath: Path) -> Tuple[str, Optional[Dict[str, Any]]]:  # noqa: C901
-    def map_metadata_value(val, nope_out=False):
-        if isinstance(val, BooleanObject):
-            val = val.value
-        elif isinstance(val, FloatObject):
-            val = float(val)
-        elif isinstance(val, NumberObject):
-            val = int(val)
-        elif isinstance(val, IndirectObject) and not nope_out:
-            # Let's make sure to nope out in case of cycles
-            val = map_metadata_value(val.getObject(), nope_out=True)
-        else:
-            val = str(val)
-        return val
-
-    metadata: Dict[str, Any] = dict()
-
-    try:
-        metadata["pdf_file_size_bytes"] = filepath.stat().st_size
-        with filepath.open("rb") as handle:
-            pdf = PdfFileReader(handle, strict=False)
-            metadata["pdf_is_encrypted"] = pdf.getIsEncrypted()
-
-        # see https://stackoverflow.com/questions/26242952/pypdf-2-decrypt-not-working
-        if metadata["pdf_is_encrypted"]:
-            pikepdf.open(filepath, allow_overwriting_input=True).save()
-
-        with filepath.open("rb") as handle:
-            pdf = PdfFileReader(handle, strict=False)
-            metadata["pdf_number_of_pages"] = pdf.getNumPages()
-            pdf_document_info = pdf.getDocumentInfo()
-
-            if pdf_document_info is None:
-                raise ValueError("PDF metadata unavailable")
-
-            for key, val in pdf_document_info.items():
-                metadata[str(key)] = map_metadata_value(val)
-
-            annots = [page.get("/Annots", []) for page in pdf.pages]
-            annots = reduce(lambda x, y: x + y, annots)
-            links = set()
-            for a in annots:
-                if isinstance(a, IndirectObject):
-                    note = a.getObject()
-                else:
-                    note = a
-                link = note.get("/A", {}).get("/URI")
-                if link:
-                    links.add(link)
-            metadata["pdf_hyperlinks"] = links
-
-    except Exception as e:
-        relative_filepath = "/".join(str(filepath).split("/")[-4:])
-        error_msg = f"Failed to read metadata of {relative_filepath}, error: {e}"
-        logger.error(error_msg)
-        return error_msg, None
-
-    return constants.RETURNCODE_OK, metadata
-
-
 def to_utc(timestamp: datetime) -> datetime:
     offset = timestamp.utcoffset()
     if offset is None:
@@ -340,613 +106,6 @@ def to_utc(timestamp: datetime) -> datetime:
     return timestamp
 
 
-# TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!!
-def search_only_headers_anssi(filepath: Path):  # noqa: C901
-    class HEADER_TYPE(Enum):
-        HEADER_FULL = 1
-        HEADER_MISSING_CERT_ITEM_VERSION = 2
-        HEADER_MISSING_PROTECTION_PROFILES = 3
-        HEADER_DUPLICITIES = 4
-
-    rules_certificate_preface = [
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur\\(s\\)(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit \\(référence/version\\)(.+)Nom de la TOE \\(référence/version\\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\\(s\\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\\(s\\)(.+)Evaluation facility(.+)Recognition arrangements",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification report reference(.+)Product name \\(reference / version\\)(.+)TOE name \\(reference / version\\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements",
-        ),
-        (
-            HEADER_TYPE.HEADER_FULL,
-            "Certification report reference(.+)TOE name(.+)Product's reference/ version(.+)TOE's reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements",
-        ),
-        # corrupted text (duplicities)
-        (
-            HEADER_TYPE.HEADER_DUPLICITIES,
-            "Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles",
-        ),
-        # rules without product version
-        (
-            HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
-            "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
-            "Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeur (.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        (
-            HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-        # rules without protection profile
-        (
-            HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES,
-            "Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d'évaluation et version(.+)Niveau d'évaluation(.+)Développeurs(.+)Centre d'évaluation(.+)Accords de reconnaissance applicables",
-        ),
-    ]
-
-    # statistics about rules success rate
-    num_rules_hits = {}
-    for rule in rules_certificate_preface:
-        num_rules_hits[rule[1]] = 0
-
-    items_found = {}  # type: ignore # noqa
-
-    try:
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath)
-
-        # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs
-        pos = whole_text.find("")
-        if pos != -1:
-            pos = whole_text.find("", pos)
-            if pos != -1:
-                whole_text = whole_text[pos:]
-
-        no_match_yet = True
-        other_rule_already_match = False
-        rule_index = -1
-        for rule in rules_certificate_preface:
-            rule_index += 1
-            rule_and_sep = rule[1] + REGEXEC_SEP
-
-            for m in re.finditer(rule_and_sep, whole_text):
-                if no_match_yet:
-                    items_found[constants.TAG_HEADER_MATCH_RULES] = []
-                    no_match_yet = False
-
-                # insert rule if at least one match for it was found
-                if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]:
-                    items_found[constants.TAG_HEADER_MATCH_RULES].append(rule[1])
-
-                if not other_rule_already_match:
-                    other_rule_already_match = True
-                else:
-                    logger.warning(f"WARNING: multiple rules are matching same certification document: {filepath}")
-
-                num_rules_hits[rule[1]] += 1  # add hit to this rule
-                match_groups = m.groups()
-                index_next_item = 0
-                items_found[constants.TAG_CERT_ID] = normalize_match_string(match_groups[index_next_item])
-                index_next_item += 1
-
-                items_found[constants.TAG_CERT_ITEM] = normalize_match_string(match_groups[index_next_item])
-                index_next_item += 1
-
-                if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION:
-                    items_found[constants.TAG_CERT_ITEM_VERSION] = ""
-                else:
-                    items_found[constants.TAG_CERT_ITEM_VERSION] = normalize_match_string(match_groups[index_next_item])
-                    index_next_item += 1
-
-                if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES:
-                    items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = ""
-                else:
-                    items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
-                        match_groups[index_next_item]
-                    )
-                    index_next_item += 1
-
-                items_found[constants.TAG_CC_VERSION] = normalize_match_string(match_groups[index_next_item])
-                index_next_item += 1
-
-                items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(match_groups[index_next_item])
-                index_next_item += 1
-
-                items_found[constants.TAG_DEVELOPER] = normalize_match_string(match_groups[index_next_item])
-                index_next_item += 1
-
-                items_found[constants.TAG_CERT_LAB] = normalize_match_string(match_groups[index_next_item])
-                index_next_item += 1
-    except Exception as e:
-        relative_filepath = "/".join(str(filepath).split("/")[-4:])
-        error_msg = f"Failed to parse ANSSI frontpage headers from {relative_filepath}; {e}"
-        logger.error(error_msg)
-        return error_msg, None
-
-    # if True:
-    #     print('# hits for rule')
-    #     sorted_rules = sorted(num_rules_hits.items(),
-    #                           key=operator.itemgetter(1), reverse=True)
-    #     used_rules = []
-    #     for rule in sorted_rules:
-    #         print('{:4d} : {}'.format(rule[1], rule[0]))
-    #         if rule[1] > 0:
-    #             used_rules.append(rule[0])
-
-    return constants.RETURNCODE_OK, items_found
-
-
-# TODO: Please refactor me. I need it so badlyyyyyy!!!
-def search_only_headers_bsi(filepath: Path):  # noqa: C901
-    LINE_SEPARATOR_STRICT = " "
-    NUM_LINES_TO_INVESTIGATE = 15
-    rules_certificate_preface = [
-        "(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)",
-        "(BSI-DSZ-CC-.+?) zu (.+?) der (.*)",
-    ]
-
-    items_found = {}  # type: ignore # noqa
-    no_match_yet = True
-
-    try:
-        # Process front page with info: cert_id, certified_item and developer
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
-        )
-
-        for rule in rules_certificate_preface:
-            rule_and_sep = rule + REGEXEC_SEP
-
-            for m in re.finditer(rule_and_sep, whole_text):
-                if no_match_yet:
-                    items_found[constants.TAG_HEADER_MATCH_RULES] = []
-                    no_match_yet = False
-
-                # insert rule if at least one match for it was found
-                if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]:
-                    items_found[constants.TAG_HEADER_MATCH_RULES].append(rule)
-
-                match_groups = m.groups()
-                cert_id = match_groups[0]
-                certified_item = match_groups[1]
-                developer = match_groups[2]
-
-                FROM_KEYWORD_LIST = [" from ", " der "]
-                for from_keyword in FROM_KEYWORD_LIST:
-                    from_keyword_len = len(from_keyword)
-                    if certified_item.find(from_keyword) != -1:
-                        logger.warning(
-                            f"string {from_keyword} detected in certified item - shall not be here, fixing..."
-                        )
-                        certified_item_first = certified_item[: certified_item.find(from_keyword)]
-                        developer = certified_item[certified_item.find(from_keyword) + from_keyword_len :]
-                        certified_item = certified_item_first
-                        continue
-
-                end_pos = developer.find("\f-")
-                if end_pos == -1:
-                    end_pos = developer.find("\fBSI")
-                if end_pos == -1:
-                    end_pos = developer.find("Bundesamt")
-                if end_pos != -1:
-                    developer = developer[:end_pos]
-
-                items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
-                items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item)
-                items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer)
-                items_found[constants.TAG_CERT_LAB] = "BSI"
-
-        # Process page with more detailed sample info
-        # PP Conformance, Functionality, Assurance
-        rules_certificate_third = ["PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified"]
-
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath)
-
-        for rule in rules_certificate_third:
-            rule_and_sep = rule + REGEXEC_SEP
-
-            for m in re.finditer(rule_and_sep, whole_text):
-                # check if previous rules had at least one match
-                if constants.TAG_CERT_ID not in items_found.keys():
-                    logger.error("ERROR: front page not found for file: {}".format(filepath))
-
-                match_groups = m.groups()
-                ref_protection_profiles = match_groups[0]
-                cc_version = match_groups[1]
-                cc_security_level = match_groups[2]
-
-                items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
-                    ref_protection_profiles
-                )
-                items_found[constants.TAG_CC_VERSION] = normalize_match_string(cc_version)
-                items_found[constants.TAG_CC_SECURITY_LEVEL] = normalize_match_string(cc_security_level)
-
-        # print('\n*** Certificates without detected preface:')
-        # for file_name in files_without_match:
-        #     print('No hits for {}'.format(file_name))
-        # print('Total no hits files: {}'.format(len(files_without_match)))
-        # print('\n**********************************')
-    except Exception as e:
-        relative_filepath = "/".join(str(filepath).split("/")[-4:])
-        error_msg = f"Failed to parse BSI headers from frontpage: {relative_filepath}; {e}"
-        logger.error(error_msg)
-        return error_msg, None
-
-    return constants.RETURNCODE_OK, items_found
-
-
-# Port from old-api branch
-def search_only_headers_nscib(filepath: Path):  # noqa: C901
-    LINE_SEPARATOR_STRICT = " "
-    NUM_LINES_TO_INVESTIGATE = 60
-    items_found: Dict[str, str] = {}
-
-    try:
-        # Process front page with info: cert_id, certified_item and developer
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
-        )
-
-        certified_item = ""
-        developer = ""
-        cert_lab = ""
-        cert_id = ""
-
-        lines = whole_text_with_newlines.splitlines()
-        no_match_yet = True
-        item_offset = -1
-
-        for line_index in range(0, len(lines)):
-            line = lines[line_index]
-
-            if "Certification Report" in line:
-                item_offset = line_index + 1
-            if "Assurance Continuity Maintenance Report" in line:
-                item_offset = line_index + 1
-
-            SPONSORDEVELOPER_STR = "Sponsor and developer:"
-
-            if SPONSORDEVELOPER_STR in line:
-                if no_match_yet:
-                    items_found = {}
-                    no_match_yet = False
-
-                # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report'
-                certified_item = ""
-                for name_index in range(item_offset, line_index):
-                    certified_item += lines[name_index] + " "
-                developer = line[line.find(SPONSORDEVELOPER_STR) + len(SPONSORDEVELOPER_STR) :]
-
-            SPONSOR_STR = "Sponsor:"
-
-            if SPONSOR_STR in line:
-                if no_match_yet:
-                    items_found = {}
-                    no_match_yet = False
-
-                # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report'
-                certified_item = ""
-                for name_index in range(item_offset, line_index):
-                    certified_item += lines[name_index] + " "
-
-            DEVELOPER_STR = "Developer:"
-            if DEVELOPER_STR in line:
-                developer = line[line.find(DEVELOPER_STR) + len(DEVELOPER_STR) :]
-
-            CERTLAB_STR = "Evaluation facility:"
-            if CERTLAB_STR in line:
-                cert_lab = line[line.find(CERTLAB_STR) + len(CERTLAB_STR) :]
-
-            REPORTNUM_STR = "Report number:"
-            if REPORTNUM_STR in line:
-                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
-
-        if not no_match_yet:
-            items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
-            items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item)
-            items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer)
-            items_found[constants.TAG_CERT_LAB] = cert_lab
-
-    except Exception as e:
-        error_msg = f"Failed to parse NSCIB headers from frontpage: {filepath}; {e}"
-        logger.error(error_msg)
-        return error_msg, None
-
-    return constants.RETURNCODE_OK, items_found
-
-
-# Port from old-api branch
-def search_only_headers_niap(filepath: Path):
-    LINE_SEPARATOR_STRICT = " "
-    NUM_LINES_TO_INVESTIGATE = 15
-    items_found: Dict[str, str] = {}
-
-    try:
-        # Process front page with info: cert_id, certified_item and developer
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
-        )
-
-        certified_item = ""
-        cert_id = ""
-
-        lines = whole_text_with_newlines.splitlines()
-        no_match_yet = True
-        item_offset = -1
-
-        for line_index in range(0, len(lines)):
-            line = lines[line_index]
-
-            if "Validation Report" in line:
-                item_offset = line_index + 1
-
-            REPORTNUM_STR = "Report Number:"
-            if REPORTNUM_STR in line:
-                if no_match_yet:
-                    items_found = {}
-                    no_match_yet = False
-
-                # all lines above till 'Certification Report' or 'Assurance Continuity Maintenance Report'
-                certified_item = ""
-                for name_index in range(item_offset, line_index):
-                    certified_item += lines[name_index] + " "
-                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
-                break
-
-        if not no_match_yet:
-            items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
-            items_found[constants.TAG_CERT_ITEM] = normalize_match_string(certified_item)
-            items_found[constants.TAG_CERT_LAB] = "US NIAP"
-
-    except Exception as e:
-        error_msg = f"Failed to parse NIAP headers from frontpage: {filepath}; {e}"
-        logger.error(error_msg)
-        return error_msg, None
-
-    return constants.RETURNCODE_OK, items_found
-
-
-# Port from old-api branch
-def search_only_headers_canada(filepath: Path):  # noqa: C901
-    LINE_SEPARATOR_STRICT = " "
-    NUM_LINES_TO_INVESTIGATE = 20
-    items_found: Dict[str, str] = {}
-    try:
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
-        )
-
-        cert_id = ""
-
-        lines = whole_text_with_newlines.splitlines()
-        no_match_yet = True
-        for line_index in range(0, len(lines)):
-            line = lines[line_index]
-            if "Government of Canada, Communications Security Establishment" in line:
-                REPORTNUM_STR1 = "Evaluation number:"
-                REPORTNUM_STR2 = "Document number:"
-                matched_number_str = ""
-                line_certid = lines[line_index + 1]
-                if line_certid.startswith(REPORTNUM_STR1):
-                    matched_number_str = REPORTNUM_STR1
-                if line_certid.startswith(REPORTNUM_STR2):
-                    matched_number_str = REPORTNUM_STR2
-                if matched_number_str != "":
-                    if no_match_yet:
-                        items_found = {}
-                        no_match_yet = False
-
-                    cert_id = line_certid[line_certid.find(matched_number_str) + len(matched_number_str) :]
-                    break
-
-            if (
-                "Government of Canada. This document is the property of the Government of Canada. It shall not be altered,"
-                in line
-            ):
-                REPORTNUM_STR = "Evaluation number:"
-                for offset in range(1, 20):
-                    line_certid = lines[line_index + offset]
-                    if "UNCLASSIFIED" in line_certid:
-                        if no_match_yet:
-                            items_found = {}
-                            no_match_yet = False
-                        line_certid = lines[line_index + offset - 4]
-                        cert_id = line_certid[line_certid.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
-                        break
-                if not no_match_yet:
-                    break
-
-            if (
-                "UNCLASSIFIED / NON CLASSIFIÉ" in line
-                and "COMMON CRITERIA CERTIFICATION REPORT" in lines[line_index + 2]
-            ):
-                line_certid = lines[line_index + 1]
-                if no_match_yet:
-                    items_found = {}
-                    no_match_yet = False
-                cert_id = line_certid
-                break
-
-        if not no_match_yet:
-            items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
-            items_found[constants.TAG_CERT_LAB] = "CANADA"
-
-    except Exception as e:
-        error_msg = f"Failed to parse Canada headers from frontpage: {filepath}; {e}"
-        logger.error(error_msg)
-        return error_msg, None
-
-    return constants.RETURNCODE_OK, items_found
-
-
-def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, int]]]]:
-    try:
-        result = parse_cert_file(filepath, cc_search_rules, -1, constants.LINE_SEPARATOR)
-
-        processed_result = {}
-        top_level_keys = list(result.keys())
-        for key in top_level_keys:
-            processed_result[key] = {key: val for key, val in gen_dict_extract(result[key])}
-
-    except Exception as e:
-        relative_filepath = "/".join(str(filepath).split("/")[-4:])
-        error_msg = f"Failed to parse keywords from: {relative_filepath}; {e}"
-        logger.error(error_msg)
-        return error_msg, None
-    return constants.RETURNCODE_OK, processed_result
-
-
-def plot_dataframe_graph(
-    data: Dict,
-    label: str,
-    file_name: str,
-    density: bool = False,
-    cumulative: bool = False,
-    bins: int = 50,
-    log: bool = True,
-    show: bool = True,
-) -> None:
-    pd_data = pd.Series(data)
-    pd_data.hist(bins=bins, label=label, density=density, cumulative=cumulative)
-    plt.savefig(file_name)
-    if show:
-        plt.show()
-
-    if log:
-        sorted_data = pd_data.value_counts(ascending=True)
-
-    logger.info(sorted_data.where(sorted_data > 1).dropna())
-
-
 def is_in_dict(target_dict: Dict, path: str) -> bool:
     current_level = target_dict
     for item in path:
@@ -957,147 +116,6 @@ def is_in_dict(target_dict: Dict, path: str) -> bool:
     return True
 
 
-def search_files(folder: str) -> Iterator[str]:
-    for root, _, files in os.walk(folder):
-        yield from [os.path.join(root, x) for x in files]
-
-
-def save_modified_cert_file(target_file: Union[str, Path], modified_cert_file_text: str, is_unicode_text: bool) -> None:
-    if is_unicode_text:
-        write_file = Path(target_file).open("w", encoding="utf8", errors="replace")
-    else:
-        write_file = Path(target_file).open("w", errors="replace")
-
-    try:
-        write_file.write(modified_cert_file_text)
-    except UnicodeEncodeError:
-        print("UnicodeDecodeError while writing file fragments back")
-    finally:
-        write_file.close()
-
-
-def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR):  # noqa: C901
-    whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-        file_name, limit_max_lines, line_separator
-    )
-
-    items_found_all = {}
-    for rule_group, rules in search_rules.items():
-        if rule_group not in items_found_all:
-            items_found_all[rule_group] = {}
-
-        items_found = items_found_all[rule_group]
-
-        for rule in rules:
-            rule_str, rule_and_sep = rule
-
-            for m in re.finditer(rule_and_sep, whole_text):
-                if rule_str not in items_found:
-                    items_found[rule_str] = {}
-
-                match = m.group()
-                match = normalize_match_string(match)
-
-                match_len = len(match)
-                if match_len > MAX_ALLOWED_MATCH_LENGTH:
-                    logger.warning(f"Excessive match with length of {match_len} detected for rule {rule_str}")
-
-                if match not in items_found[rule_str]:
-                    items_found[rule_str][match] = {}
-                    items_found[rule_str][match][TAG_MATCH_COUNTER] = 0
-                    if APPEND_DETAILED_MATCH_MATCHES:
-                        items_found[rule_str][match][TAG_MATCH_MATCHES] = []
-                items_found[rule_str][match][TAG_MATCH_COUNTER] += 1
-                match_span = m.span()
-                if APPEND_DETAILED_MATCH_MATCHES:
-                    items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1]])
-
-    return items_found_all
-
-
-def normalize_match_string(match: str) -> str:
-    match = match.strip().rstrip('];.”":)(,').rstrip(os.sep).replace("  ", " ")
-    return "".join(filter(str.isprintable, match))
-
-
-def load_cert_file(
-    file_name: Union[str, Path], limit_max_lines: int = -1, line_separator: str = LINE_SEPARATOR
-) -> Tuple[str, str, bool]:
-    lines = []
-    was_unicode_decode_error = False
-    with Path(file_name).open("r", errors=FILE_ERRORS_STRATEGY) as f:
-        try:
-            lines = f.readlines()
-        except UnicodeDecodeError:
-            f.close()
-            was_unicode_decode_error = True
-            print("  WARNING: UnicodeDecodeError, opening as utf8")
-
-            with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
-                # coding failure, try line by line
-                line = " "
-                while line:
-                    try:
-                        line = f2.readline()
-                        lines.append(line)
-                    except UnicodeDecodeError:
-                        # ignore error
-                        continue
-
-    whole_text = ""
-    whole_text_with_newlines = ""
-    # we will estimate the line for searched matches
-    # => we need to known how much lines were modified (removal of eoln..)
-    # for removed newline and for any added separator
-    # line_length_compensation = 1 - len(LINE_SEPARATOR)
-    lines_included = 0
-    for line in lines:
-        if limit_max_lines != -1 and lines_included >= limit_max_lines:
-            break
-
-        whole_text_with_newlines += line
-        line = line.replace("\n", "")
-        whole_text += line
-        whole_text += line_separator
-        lines_included += 1
-
-    return whole_text, whole_text_with_newlines, was_unicode_decode_error
-
-
-def load_cert_html_file(file_name: str) -> str:
-    with open(file_name, "r", errors=FILE_ERRORS_STRATEGY) as f:
-        try:
-            whole_text = f.read()
-        except UnicodeDecodeError:
-            f.close()
-            with open(file_name, "r", encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
-                try:
-                    whole_text = f2.read()
-                except UnicodeDecodeError:
-                    print("### ERROR: failed to read file {}".format(file_name))
-    return whole_text
-
-
-def gen_dict_extract(dct: Dict, searched_key: Hashable = "count") -> Generator[Any, None, None]:
-    """
-    Function to flatten dictionary with some serious limitations. We only expect to use it temporarily on dictionary
-    produced by extract_keywords that contains many layers. On the deepest level in that dictionary, 'some_match': {'count': frequency}.
-    The output of the function will be list of tuples ('some_match': frequency)
-    :param searched_key: key to search, 'count'
-    :param dct: Dictionary to search
-    :return: List of tuples
-    """
-    for key, value in dct.items():
-        if key == searched_key:
-            yield value
-        if isinstance(value, dict):
-            for result in gen_dict_extract(value, searched_key):
-                if isinstance(result, tuple):
-                    yield result
-                else:
-                    yield key, result
-
-
 def compute_heuristics_version(cert_name: str) -> Set[str]:
     """
     Will extract possible versions from the name of sample
diff --git a/sec_certs/utils/pandas_helpers.py b/sec_certs/utils/pandas.py
index 4324657d..b6a2fe9e 100644
--- a/sec_certs/utils/pandas_helpers.py
+++ b/sec_certs/utils/pandas.py
@@ -4,19 +4,23 @@ import copy
 import tempfile
 import xml.etree.ElementTree as ET
 import zipfile
+import logging
 from dataclasses import dataclass
 from pathlib import Path
 from shutil import copyfile
-from typing import Any, Final, List, Optional, Set, Tuple, Union
+from typing import Any, Final, List, Optional, Set, Tuple, Union, Dict
 
 import numpy as np
 import pandas as pd
+from matplotlib import pyplot as plt
 from tqdm.notebook import tqdm
 
 from sec_certs.utils import helpers
 from sec_certs.dataset.cve import CVEDataset
 from sec_certs.sample.sar import SAR
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass(eq=True, frozen=True)
 class SecondarySFPCluster:
@@ -435,3 +439,25 @@ def move_fixing_mu_to_directory(
         copyfile(inpath / i, Path(outdir) / i)
 
     return mu_filenames
+
+
+def plot_dataframe_graph(
+    data: Dict,
+    label: str,
+    file_name: str,
+    density: bool = False,
+    cumulative: bool = False,
+    bins: int = 50,
+    log: bool = True,
+    show: bool = True,
+) -> None:
+    pd_data = pd.Series(data)
+    pd_data.hist(bins=bins, label=label, density=density, cumulative=cumulative)
+    plt.savefig(file_name)
+    if show:
+        plt.show()
+
+    if log:
+        sorted_data = pd_data.value_counts(ascending=True)
+
+    logger.info(sorted_data.where(sorted_data > 1).dropna())
\ No newline at end of file
diff --git a/sec_certs/utils/pdf.py b/sec_certs/utils/pdf.py
new file mode 100644
index 00000000..081fce37
--- /dev/null
+++ b/sec_certs/utils/pdf.py
@@ -0,0 +1,138 @@
+import logging
+from pathlib import Path
+from typing import Tuple, Optional, Dict, Any
+from datetime import datetime, timedelta, timezone
+from functools import reduce
+
+import pdftotext
+import pikepdf
+from PyPDF2 import PdfFileReader
+from PyPDF2.generic import BooleanObject, FloatObject, NumberObject, IndirectObject
+
+from sec_certs import constants as constants
+
+logger = logging.getLogger(__name__)
+
+
+def repair_pdf(file: Path) -> None:
+    """
+    Some pdfs can't be opened by PyPDF2 - opening them with pikepdf and then saving them fixes this issue.
+    By opening this file in a pdf reader, we can already extract number of pages
+    :param file: file name
+    :return: number of pages in pdf file
+    """
+    pdf = pikepdf.Pdf.open(file, allow_overwriting_input=True)
+    pdf.save(file)
+
+
+def convert_pdf_file(pdf_path: Path, txt_path: Path) -> str:
+    try:
+        with pdf_path.open("rb") as pdf_handle:
+            pdf = pdftotext.PDF(pdf_handle, "", True)  # No password, Raw=True
+            txt = "".join(pdf)
+    except Exception as e:
+        logger.error(f"Error when converting pdf->txt: {e}")
+        return constants.RETURNCODE_NOK
+
+    with txt_path.open("w", encoding="utf-8") as txt_handle:
+        txt_handle.write(txt)
+
+    return constants.RETURNCODE_OK
+
+
+def parse_pdf_date(dateval) -> Optional[datetime]:
+    """
+    Parse PDF metadata date format:
+
+    ```
+        parse_pdf_date(b"D:20110617082321-04'00'")
+    ```
+    into
+    ```
+        datetime.datetime(2011, 6, 17, 8, 23, 21, tzinfo=datetime.timezone(datetime.timedelta(days=-1, seconds=72000)))
+    ```
+    """
+    if dateval is None:
+        return None
+    clean = dateval.decode("utf-8").replace("D:", "")
+    tz = None
+    tzoff = None
+    if "+" in clean:
+        clean, tz = clean.split("+")
+        tzoff = 1
+    if "-" in clean:
+        clean, tz = clean.split("-")
+        tzoff = -1
+    elif "Z" in clean:
+        clean, tz = clean.split("Z")
+        tzoff = 1
+    try:
+        res_datetime = datetime.strptime(clean, "%Y%m%d%H%M%S")
+        if tz and tzoff:
+            tz_datetime = datetime.strptime(tz, "%H'%M'")
+            delta = tzoff * timedelta(hours=tz_datetime.hour, minutes=tz_datetime.minute)
+            res_tz = timezone(delta)
+            res_datetime = res_datetime.replace(tzinfo=res_tz)
+        return res_datetime
+    except ValueError:
+        return None
+
+
+def extract_pdf_metadata(filepath: Path) -> Tuple[str, Optional[Dict[str, Any]]]:  # noqa: C901
+    def map_metadata_value(val, nope_out=False):
+        if isinstance(val, BooleanObject):
+            val = val.value
+        elif isinstance(val, FloatObject):
+            val = float(val)
+        elif isinstance(val, NumberObject):
+            val = int(val)
+        elif isinstance(val, IndirectObject) and not nope_out:
+            # Let's make sure to nope out in case of cycles
+            val = map_metadata_value(val.getObject(), nope_out=True)
+        else:
+            val = str(val)
+        return val
+
+    metadata: Dict[str, Any] = dict()
+
+    try:
+        metadata["pdf_file_size_bytes"] = filepath.stat().st_size
+        with filepath.open("rb") as handle:
+            pdf = PdfFileReader(handle, strict=False)
+            metadata["pdf_is_encrypted"] = pdf.getIsEncrypted()
+
+        # see https://stackoverflow.com/questions/26242952/pypdf-2-decrypt-not-working
+        if metadata["pdf_is_encrypted"]:
+            pikepdf.open(filepath, allow_overwriting_input=True).save()
+
+        with filepath.open("rb") as handle:
+            pdf = PdfFileReader(handle, strict=False)
+            metadata["pdf_number_of_pages"] = pdf.getNumPages()
+            pdf_document_info = pdf.getDocumentInfo()
+
+            if pdf_document_info is None:
+                raise ValueError("PDF metadata unavailable")
+
+            for key, val in pdf_document_info.items():
+                metadata[str(key)] = map_metadata_value(val)
+
+            annots = [page.get("/Annots", []) for page in pdf.pages]
+            annots = reduce(lambda x, y: x + y, annots)
+            links = set()
+            for a in annots:
+                if isinstance(a, IndirectObject):
+                    note = a.getObject()
+                else:
+                    note = a
+                link = note.get("/A", {}).get("/URI")
+                if link:
+                    links.add(link)
+            metadata["pdf_hyperlinks"] = links
+
+    except Exception as e:
+        relative_filepath = "/".join(str(filepath).split("/")[-4:])
+        error_msg = f"Failed to read metadata of {relative_filepath}, error: {e}"
+        logger.error(error_msg)
+        return error_msg, None
+
+    return constants.RETURNCODE_OK, metadata
\ No newline at end of file
diff --git a/sec_certs/utils/sanitization.py b/sec_certs/utils/sanitization.py
new file mode 100644
index 00000000..37a59051
--- /dev/null
+++ b/sec_certs/utils/sanitization.py
@@ -0,0 +1,43 @@
+import html
+import logging
+from datetime import date
+from typing import Optional, Union, Set
+
+import numpy as np
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def sanitize_link(record: Optional[str]) -> Optional[str]:
+    if not record:
+        return None
+    return record.replace(":443", "").replace(" ", "%20").replace("http://", "https://")
+
+
+def sanitize_date(record: Union[pd.Timestamp, date, np.datetime64]) -> Union[date, None]:
+    if pd.isnull(record):
+        return None
+    elif isinstance(record, pd.Timestamp):
+        return record.date()
+    elif isinstance(record, (date, type(None))):
+        return record
+    raise ValueError("Unsupported type given as input")
+
+
+def sanitize_string(record: str) -> str:
+    # There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT &#x2f; AT90SC12836RCFT rev. I &amp;&#x23;38&#x3b; J' that has to be unescaped twice
+    string = html.unescape(html.unescape(record)).replace("\n", "")
+    return " ".join(string.split())
+
+
+def sanitize_security_levels(record: Union[str, Set[str]]) -> Set[str]:
+    if isinstance(record, str):
+        record = set(record.split(","))
+    return record - {"Basic", "ND-PP", "PP\xa0Compliant", "None"}
+
+
+def sanitize_protection_profiles(record: str) -> list:
+    if not record:
+        return []
+    return record.split(",")
\ No newline at end of file
diff --git a/sec_certs/utils/tables.py b/sec_certs/utils/tables.py
new file mode 100644
index 00000000..5c988a1b
--- /dev/null
+++ b/sec_certs/utils/tables.py
@@ -0,0 +1,63 @@
+import re
+import logging
+from pathlib import Path
+from typing import Set, List, Optional, Union
+
+logger = logging.getLogger(__name__)
+
+
+def parse_list_of_tables(txt: str) -> Set[str]:
+    """
+    Parses list of tables from function find_tables(), finds ones that mention algorithms
+    :param txt: chunk of text
+    :return: set of all pages mentioning algorithm table
+    """
+    rr = re.compile(r"^.+?(?:[Ff]unction|[Aa]lgorithm|[Ss]ecurity [Ff]unctions?).+?(?P<page_num>\d+)$", re.MULTILINE)
+    pages = set()
+    for m in rr.finditer(txt):
+        pages.add(m.group("page_num"))
+    return pages
+
+
+def find_tables_iterative(file_text: str) -> List[int]:
+    current_page = 1
+    pages = set()
+    for line in file_text.split("\n"):
+        if "\f" in line:
+            current_page += 1
+        if line.startswith("Table ") or line.startswith("Exhibit"):
+            pages.add(current_page)
+            pages.add(current_page + 1)
+            if current_page > 2:
+                pages.add(current_page - 1)
+    if not pages:
+        logger.warning("No pages found")
+    for page in pages:
+        if page > current_page - 1:
+            return list(pages - {page})
+
+    return list(pages)
+
+
+def find_tables(txt: str, file_name: Path) -> Optional[Union[List[str], List[int]]]:
+    """
+    Function that tries to pages in security policy pdf files, where it's possible to find a table containing
+    algorithms
+    :param txt: file in .txt format (output of pdftotext)
+    :param file_name: name of the file
+    :return:    list of pages possibly containing a table
+                None if these cannot be found
+    """
+    # Look for "List of Tables", where we can find exactly tables with page num
+    tables_regex = re.compile(r"^(?:(?:[Tt]able\s|[Ll]ist\s)(?:[Oo]f\s))[Tt]ables[\s\S]+?\f", re.MULTILINE)
+    table = tables_regex.search(txt)
+    if table:
+        rb = parse_list_of_tables(table.group())
+        if rb:
+            return list(rb)
+        return None
+
+    # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer
+    logger.info(f"parsing tables in {file_name}")
+    table_page_indices = find_tables_iterative(txt)
+    return table_page_indices if table_page_indices else None
\ No newline at end of file