Fix most test issues caused by new changes.

11 files changed, 161 insertions, 294 deletions

diff --git a/sec_certs/config/settings-schema.json b/sec_certs/config/settings-schema.json
index 7fcec5d6..c4871096 100644
--- a/sec_certs/config/settings-schema.json
+++ b/sec_certs/config/settings-schema.json
@@ -88,9 +88,6 @@
                 }
             ]
         },
-        "use_text_with_newlines_during_parsing": {
-            "$ref": "#/definitions/settings_boolean_entry"
-        },
         "n_threads": {
             "allOf": [
                 {
@@ -168,7 +165,6 @@
         "log_filepath",
         "smallest_certificate_id_to_connect",
         "year_difference_between_validations",
-        "use_text_with_newlines_during_parsing",
         "n_threads",
         "cpe_matching_threshold",
         "cpe_n_max_matches",
diff --git a/sec_certs/config/settings.yaml b/sec_certs/config/settings.yaml
index b0228238..6a7f9d6b 100644
--- a/sec_certs/config/settings.yaml
+++ b/sec_certs/config/settings.yaml
@@ -12,9 +12,6 @@ year_difference_between_validations:
     During validation we don't connect certificates with validation dates
     difference higher than _this_
   value: 7
-use_text_with_newlines_during_parsing:
-  description: During keyword search, search in text with newlines
-  value: true
 n_threads:
   description: How many threads to use for parallel computations
   value: 8
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index 7ec6cc13..a2091f97 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -278,12 +278,12 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType):
 
             all_cert_ids.add(cert_id)
 
-            # ['keywords_scan', 'rules_cert_id']
+            # ['report.keywords_scan', 'cc_cert_id']
             all_cert_ids.update(cert_obj.pdf_data.keywords_rules_cert_id)
 
-            # ['st_keywords_scan']['rules_cert_id']
+            # ['st.keywords_scan']['cc_cert_id']
             if cert_obj.pdf_data.st_keywords is not None:
-                all_cert_ids.update(cert_obj.pdf_data.st_keywords["rules_cert_id"])
+                all_cert_ids.update(cert_obj.pdf_data.st_keywords["cc_cert_id"])
 
         return all_cert_ids
 
@@ -847,7 +847,10 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType):
                 kws = getattr(cert.pdf_data, kw_attr)
                 if not kws:
                     return set()
-                return set(kws["rules_cert_id"].keys())
+                res = set()
+                for scheme, matches in kws["cc_cert_id"].items():
+                    res.update(matches.keys())
+                return res
 
             return func
 
diff --git a/sec_certs/model/sar_transformer.py b/sec_certs/model/sar_transformer.py
index 58f96c0b..d87241b5 100644
--- a/sec_certs/model/sar_transformer.py
+++ b/sec_certs/model/sar_transformer.py
@@ -115,7 +115,7 @@ class SARTransformer(BaseEstimator, TransformerMixin):
         return set(final_candidates.values()) if final_candidates else None
 
     @staticmethod
-    def _parse_sar_dict(dct: Dict[str, int], dgst: str) -> Set[SAR]:
+    def _parse_sar_dict(dct: Dict[str, Dict[str, int]], dgst: str) -> Set[SAR]:
         """
         Accepts st_keywords or report_keywords dictionary. Will reconstruct SAR objects from it. Each SAR family can
         appear multiple times in the dictionary (due to conflicts) with different levels. Iterated item will replace
@@ -124,26 +124,28 @@ class SARTransformer(BaseEstimator, TransformerMixin):
 
         Only SARs with recovered level are considered, e.g. ASE_REQ.2 is valid string while ASE_REQ is not.
 
-        :param Dict[str, int] dct: _description_
-        :return Optional[Set[SAR]]: _description_
+        :param dct: _description_
+        :param dgst: DIgest of the processed certificate.
+        :return: _description_
         """
         sars: Dict[str, Tuple[SAR, int]] = dict()
-        for sar_string, n_occurences in dct.items():
-            try:
-                candidate = SAR.from_string(sar_string)
-            except ValueError as e:
-                logger.debug(f"Badly formatted SAR string {sar_string}, skipping: {e}")
-                continue
+        for sar_class, class_matches in dct.items():
+            for sar_string, n_occurences in class_matches.items():
+                try:
+                    candidate = SAR.from_string(sar_string)
+                except ValueError as e:
+                    logger.debug(f"Badly formatted SAR string {sar_string}, skipping: {e}")
+                    continue
 
-            if candidate.family in sars:
-                logger.debug(
-                    f"Cert {dgst} Attempting to add {candidate} while {sars[candidate.family]}  already in SARS"
-                )
+                if candidate.family in sars:
+                    logger.debug(
+                        f"Cert {dgst} Attempting to add {candidate} while {sars[candidate.family]}  already in SARS"
+                    )
 
-            if (candidate.family not in sars) or (
-                candidate.family in sars and candidate.level > sars[candidate.family][0].level
-            ):
-                sars[candidate.family] = (candidate, n_occurences)
+                if (candidate.family not in sars) or (
+                    candidate.family in sars and candidate.level > sars[candidate.family][0].level
+                ):
+                    sars[candidate.family] = (candidate, n_occurences)
         return {x[0] for x in sars.values()} if sars else set()
 
     @staticmethod
diff --git a/sec_certs/rules.yaml b/sec_certs/rules.yaml
index 407287f6..ae860702 100644
--- a/sec_certs/rules.yaml
+++ b/sec_certs/rules.yaml
@@ -25,7 +25,7 @@ cc_cert_id:
     - "NSCIB-CC-[0-9]+-MA[0-9]*"  # Netherlands (new number NSCIB-CC-222073-MA NSCIB-CC-200716-MA2)
     - "NSCIB-CC-[0-9][0-9]-[0-9]+"  # Netherlands   (old number NSCIB-CC-05-6609)
     - "NSCIB-CC-[0-9][0-9]-[0-9]+-CR[0-9]+"  # Netherlands (NSCIB-CC-year2digits-number-CR)
-  NO:
+  "NO":
     - "SERTIT-[0-9]+"  # Norway
   US:
     - "CCEVS-VR-(?:|VID)[0-9]+-[0-9]+[a-z]?"  # US NSA (CCEVS-VR-10884-2018 CCEVS-VR-VID10877-2018)
diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py
index 27326022..a26f4310 100644
--- a/sec_certs/sample/common_criteria.py
+++ b/sec_certs/sample/common_criteria.py
@@ -19,7 +19,7 @@ import sec_certs.utils.pdf
 import sec_certs.utils.sanitization
 from sec_certs import constants as constants
 from sec_certs.utils import helpers
-from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, security_level_csv_scan
+from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL, security_level_csv_scan, cc_rules
 from sec_certs.sample.certificate import Certificate, Heuristics, References, logger
 from sec_certs.sample.protection_profile import ProtectionProfile
 from sec_certs.sample.sar import SAR
@@ -281,7 +281,7 @@ class CommonCriteriaCert(
 
         @property
         def keywords_rules_cert_id(self) -> Optional[Dict[str, Optional[Dict[str, Dict[str, int]]]]]:
-            return self.report_keywords.get("rules_cert_id", None) if self.report_keywords else None
+            return self.report_keywords.get("cc_cert_id", None) if self.report_keywords else None
 
         @property
         def keywords_cert_id(self) -> Optional[str]:
@@ -838,9 +838,11 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: certificate to extract the keywords for.
         :return CommonCriteriaCert: the modified certificate with extracted keywords.
         """
-        response, cert.pdf_data.report_keywords = sec_certs.utils.extract.extract_keywords(cert.state.report_txt_path)
-        if response != constants.RETURNCODE_OK:
+        report_keywords = sec_certs.utils.extract.extract_keywords(cert.state.report_txt_path, cc_rules)
+        if report_keywords is None:
             cert.state.report_extract_ok = False
+        else:
+            cert.pdf_data.report_keywords = report_keywords
         return cert
 
     @staticmethod
@@ -852,10 +854,11 @@ class CommonCriteriaCert(
         :param CommonCriteriaCert cert: certificate to extract the keywords for.
         :return CommonCriteriaCert: the modified certificate with extracted keywords.
         """
-        response, cert.pdf_data.st_keywords = sec_certs.utils.extract.extract_keywords(cert.state.st_txt_path)
-        if response != constants.RETURNCODE_OK:
+        st_keywords = sec_certs.utils.extract.extract_keywords(cert.state.st_txt_path, cc_rules)
+        if st_keywords is None:
             cert.state.st_extract_ok = False
-            cert.state.errors.append(response)
+        else:
+            cert.pdf_data.st_keywords = st_keywords
         return cert
 
     def compute_heuristics_version(self) -> None:
diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py
index 0207e5ff..650afc7a 100644
--- a/sec_certs/sample/fips.py
+++ b/sec_certs/sample/fips.py
@@ -21,7 +21,7 @@ from sec_certs.cert_rules import fips_rules
 from sec_certs.config.configuration import config
 from sec_certs.constants import LINE_SEPARATOR
 from sec_certs.utils.helpers import fips_dgst
-from sec_certs.utils.extract import save_modified_cert_file, normalize_match_string, load_cert_file
+from sec_certs.utils.extract import save_modified_cert_file, normalize_match_string, load_text_file
 from sec_certs.sample.certificate import Certificate, Heuristics, References, logger
 from sec_certs.sample.cpe import CPE
 from sec_certs.serialization.json import ComplexSerializableType
@@ -625,43 +625,13 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
         return cert
 
     @staticmethod
-    def _declare_state(text: str) -> bool:
-        """
-        If less then half of the text is formed of alphabet characters,
-        we declare the security policy as "non-parsable"
-        :param text: security policy content
-        :return: True if parsable, otherwise False
-        """
-        return len(text) * 0.5 <= len("".join(filter(str.isalpha, text)))
-
-    @staticmethod
     def find_keywords(cert: FIPSCertificate) -> Tuple[Optional[Dict], FIPSCertificate]:
         if not cert.state.txt_state:
             return None, cert
 
-        text, text_with_newlines, unicode_error = load_cert_file(
-            cert.state.sp_path.with_suffix(".pdf.txt"), -1, LINE_SEPARATOR
-        )
-
-        text_to_parse = text_with_newlines if config.use_text_with_newlines_during_parsing else text
-
-        cert.state.txt_state = FIPSCertificate._declare_state(text)
-
-        if config.ignore_first_page:
-            text_to_parse = text_to_parse[text_to_parse.index("") :]
-
-        items_found, fips_text = FIPSCertificate._parse_cert_file(FIPSCertificate._remove_platforms(text_to_parse))
-
-        save_modified_cert_file(cert.state.fragment_path.with_suffix(".fips.txt"), fips_text, unicode_error)
-
-        common_items_found, common_text = FIPSCertificate._parse_cert_file_common(
-            text_to_parse, text_with_newlines, fips_common_rules
-        )
-
-        save_modified_cert_file(cert.state.fragment_path.with_suffix(".common.txt"), common_text, unicode_error)
-        items_found.update(common_items_found)
+        keywords = sec_certs.utils.extract.extract_keywords(cert.state.sp_path.with_suffix(".pdf.txt"), fips_rules)
 
-        return items_found, cert
+        return keywords, cert
 
     @staticmethod
     def match_web_algs_to_pdf(cert: FIPSCertificate) -> int:
@@ -702,103 +672,6 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
         return text_to_parse
 
     @staticmethod
-    def _highlight_matches(items_found_all: Dict, whole_text_with_newlines: str) -> str:
-        all_matches = []
-        for rule_group in items_found_all.keys():
-            items_found = items_found_all[rule_group]
-            for rule in items_found.keys():
-                for match in items_found[rule]:
-                    all_matches.append(match)
-
-            # if AES string is removed before AES-128, -128 would be left in text => sort by length first
-            # sort before replacement based on the length of match
-            all_matches.sort(key=len, reverse=True)
-            for match in all_matches:
-                whole_text_with_newlines = whole_text_with_newlines.replace(match, "x" * len(match))
-
-        return whole_text_with_newlines
-
-    @staticmethod
-    def _process_match(rule: Pattern, items_found: Dict, rule_str: str, m: Match[str]) -> None:
-        # insert rule if at least one match for it was found
-        if rule not in items_found:
-            items_found[rule_str] = {}
-
-        match = m.group()
-        match = normalize_match_string(match)
-
-        match_len = len(match)
-        if match_len > constants.MAX_ALLOWED_MATCH_LENGTH:
-            logger.warning("Excessive match with length of {} detected for rule {}".format(match_len, rule))
-
-        if match not in items_found[rule_str]:
-            items_found[rule_str][match] = {}
-            items_found[rule_str][match][constants.TAG_MATCH_COUNTER] = 0
-            if constants.APPEND_DETAILED_MATCH_MATCHES:
-                items_found[rule_str][match][constants.TAG_MATCH_MATCHES] = []
-
-        items_found[rule_str][match][constants.TAG_MATCH_COUNTER] += 1
-        match_span = m.span()
-
-        if constants.APPEND_DETAILED_MATCH_MATCHES:
-            items_found[rule_str][match][constants.TAG_MATCH_MATCHES].append([match_span[0], match_span[1]])
-
-    @staticmethod
-    def _parse_cert_file_common(
-        text_to_parse: str, whole_text_with_newlines: str, search_rules: Dict
-    ) -> Tuple[Dict[Pattern, Dict], str]:
-        # apply all rules
-        items_found_all: Dict[Pattern, Dict] = {}
-        for rule_group, rules in search_rules.items():
-            if rule_group not in items_found_all:
-                items_found_all[rule_group] = {}
-
-            items_found = items_found_all[rule_group]
-
-            for rule_str, rule in rules:
-                for m in re.finditer(rule, text_to_parse):
-                    FIPSCertificate._process_match(rule, items_found, rule_str, m)
-
-        # highlight all found strings (by xxxxx) from the input text and store the rest
-
-        whole_text_with_newlines = FIPSCertificate._highlight_matches(items_found_all, whole_text_with_newlines)
-
-        return items_found_all, whole_text_with_newlines
-
-    @staticmethod
-    def _parse_cert_file(text_to_parse: str) -> Tuple[Dict[Pattern, Dict], str]:
-        # apply all rules
-        items_found_all: Dict = {}
-
-        for rule_group, rules in fips_rules.items():
-            if rule_group not in items_found_all:
-                items_found_all[rule_group] = {}
-
-            items_found: Dict[str, Dict] = items_found_all[rule_group]
-
-            for rule_str, rule in rules:
-                for m in rule.finditer(text_to_parse):
-                    # insert rule if at least one match for it was found
-                    if rule_str not in items_found:
-                        items_found[rule_str] = {}
-
-                    match = m.group()
-                    match = normalize_match_string(match)
-
-                    if match == "":
-                        continue
-
-                    if match not in items_found[rule_str]:
-                        items_found[rule_str][match] = {}
-                        items_found[rule_str][match][constants.TAG_MATCH_COUNTER] = 0
-
-                    items_found[rule_str][match][constants.TAG_MATCH_COUNTER] += 1
-
-                    text_to_parse = text_to_parse.replace(match, "x" * len(match))
-
-        return items_found_all, text_to_parse
-
-    @staticmethod
     def analyze_tables(tup: Tuple[FIPSCertificate, bool]) -> Tuple[bool, FIPSCertificate, List]:
         """
         Searches for tables in pdf documents of the instance.
@@ -880,27 +753,29 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
         if not self.pdf_scan.keywords:
             return
 
-        self.heuristics.keywords = copy.deepcopy(self.pdf_scan.keywords)
-        # TODO figure out why can't I delete this
-        if self.web_scan.mentioned_certs:
-            for item, value in self.web_scan.mentioned_certs.items():
-                self.heuristics.keywords["rules_cert_id"].update({"caveat_item": {item: value}})
-
-        alg_set = self._create_alg_set()
-
-        for rule in self.heuristics.keywords["rules_cert_id"]:
-            to_pop = set()
-            rr = re.compile(rule)
-            for cert in self.heuristics.keywords["rules_cert_id"][rule]:
-                if cert in alg_set:
-                    to_pop.add(cert)
-                    continue
-                self._process_to_pop(rr, cert, to_pop)
-
-            for r in to_pop:
-                self.heuristics.keywords["rules_cert_id"][rule].pop(r, None)
-
-            self.heuristics.keywords["rules_cert_id"][rule].pop(self.cert_id, None)
+        # XXX: What is this mess?
+        #
+        # self.heuristics.keywords = copy.deepcopy(self.pdf_scan.keywords)
+        # # TODO figure out why can't I delete this
+        # if self.web_scan.mentioned_certs:
+        #     for item, value in self.web_scan.mentioned_certs.items():
+        #         self.heuristics.keywords["rules_cert_id"].update({"caveat_item": {item: value}})
+        #
+        # alg_set = self._create_alg_set()
+        #
+        # for rule in self.heuristics.keywords["rules_cert_id"]:
+        #     to_pop = set()
+        #     rr = re.compile(rule)
+        #     for cert in self.heuristics.keywords["rules_cert_id"][rule]:
+        #         if cert in alg_set:
+        #             to_pop.add(cert)
+        #             continue
+        #         self._process_to_pop(rr, cert, to_pop)
+        #
+        #     for r in to_pop:
+        #         self.heuristics.keywords["rules_cert_id"][rule].pop(r, None)
+        #
+        #     self.heuristics.keywords["rules_cert_id"][rule].pop(self.cert_id, None)
 
     @staticmethod
     def get_compare(vendor: str) -> str:
diff --git a/sec_certs/sample/sar.py b/sec_certs/sample/sar.py
index 0563222e..96c8f48d 100644
--- a/sec_certs/sample/sar.py
+++ b/sec_certs/sample/sar.py
@@ -19,7 +19,7 @@ SAR_CLASS_MAPPING = {
 }
 
 SAR_CLASSES = {x for x in SAR_CLASS_MAPPING}
-SAR_DICT_KEY = "rules_security_assurance_components"
+SAR_DICT_KEY = "cc_sar"
 
 
 @dataclass(frozen=True, eq=True)
diff --git a/sec_certs/utils/extract.py b/sec_certs/utils/extract.py
index 621c9619..49126587 100644
--- a/sec_certs/utils/extract.py
+++ b/sec_certs/utils/extract.py
@@ -3,18 +3,15 @@ import os
 import re
 from enum import Enum
 from pathlib import Path
+from collections import Counter
 from typing import Any, Dict, Generator, Hashable, Iterator, Optional, Tuple, Union
 
 from sec_certs import constants as constants
 from sec_certs.cert_rules import REGEXEC_SEP
-from sec_certs.cert_rules import cc_rules as cc_search_rules
 from sec_certs.constants import (
-    APPEND_DETAILED_MATCH_MATCHES,
     FILE_ERRORS_STRATEGY,
     LINE_SEPARATOR,
-    MAX_ALLOWED_MATCH_LENGTH,
-    TAG_MATCH_COUNTER,
-    TAG_MATCH_MATCHES,
+    MAX_ALLOWED_MATCH_LENGTH
 )
 
 logger = logging.getLogger(__name__)
@@ -202,7 +199,7 @@ def search_only_headers_anssi(filepath: Path):  # noqa: C901
     items_found = {}  # type: ignore # noqa
 
     try:
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath)
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath)
 
         # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs
         pos = whole_text.find("")
@@ -299,7 +296,7 @@ def search_only_headers_bsi(filepath: Path):  # noqa: C901
 
     try:
         # Process front page with info: cert_id, certified_item and developer
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(
             filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
         )
 
@@ -328,7 +325,7 @@ def search_only_headers_bsi(filepath: Path):  # noqa: C901
                             f"string {from_keyword} detected in certified item - shall not be here, fixing..."
                         )
                         certified_item_first = certified_item[: certified_item.find(from_keyword)]
-                        developer = certified_item[certified_item.find(from_keyword) + from_keyword_len :]
+                        developer = certified_item[certified_item.find(from_keyword) + from_keyword_len:]
                         certified_item = certified_item_first
                         continue
 
@@ -349,7 +346,7 @@ def search_only_headers_bsi(filepath: Path):  # noqa: C901
         # PP Conformance, Functionality, Assurance
         rules_certificate_third = ["PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified"]
 
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(filepath)
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(filepath)
 
         for rule in rules_certificate_third:
             rule_and_sep = rule + REGEXEC_SEP
@@ -392,7 +389,7 @@ def search_only_headers_nscib(filepath: Path):  # noqa: C901
 
     try:
         # Process front page with info: cert_id, certified_item and developer
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(
             filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
         )
 
@@ -424,7 +421,7 @@ def search_only_headers_nscib(filepath: Path):  # noqa: C901
                 certified_item = ""
                 for name_index in range(item_offset, line_index):
                     certified_item += lines[name_index] + " "
-                developer = line[line.find(SPONSORDEVELOPER_STR) + len(SPONSORDEVELOPER_STR) :]
+                developer = line[line.find(SPONSORDEVELOPER_STR) + len(SPONSORDEVELOPER_STR):]
 
             SPONSOR_STR = "Sponsor:"
 
@@ -440,15 +437,15 @@ def search_only_headers_nscib(filepath: Path):  # noqa: C901
 
             DEVELOPER_STR = "Developer:"
             if DEVELOPER_STR in line:
-                developer = line[line.find(DEVELOPER_STR) + len(DEVELOPER_STR) :]
+                developer = line[line.find(DEVELOPER_STR) + len(DEVELOPER_STR):]
 
             CERTLAB_STR = "Evaluation facility:"
             if CERTLAB_STR in line:
-                cert_lab = line[line.find(CERTLAB_STR) + len(CERTLAB_STR) :]
+                cert_lab = line[line.find(CERTLAB_STR) + len(CERTLAB_STR):]
 
             REPORTNUM_STR = "Report number:"
             if REPORTNUM_STR in line:
-                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
+                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR):]
 
         if not no_match_yet:
             items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
@@ -472,7 +469,7 @@ def search_only_headers_niap(filepath: Path):
 
     try:
         # Process front page with info: cert_id, certified_item and developer
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(
             filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
         )
 
@@ -499,7 +496,7 @@ def search_only_headers_niap(filepath: Path):
                 certified_item = ""
                 for name_index in range(item_offset, line_index):
                     certified_item += lines[name_index] + " "
-                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
+                cert_id = line[line.find(REPORTNUM_STR) + len(REPORTNUM_STR):]
                 break
 
         if not no_match_yet:
@@ -521,7 +518,7 @@ def search_only_headers_canada(filepath: Path):  # noqa: C901
     NUM_LINES_TO_INVESTIGATE = 20
     items_found: Dict[str, str] = {}
     try:
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(
             filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT
         )
 
@@ -545,12 +542,12 @@ def search_only_headers_canada(filepath: Path):  # noqa: C901
                         items_found = {}
                         no_match_yet = False
 
-                    cert_id = line_certid[line_certid.find(matched_number_str) + len(matched_number_str) :]
+                    cert_id = line_certid[line_certid.find(matched_number_str) + len(matched_number_str):]
                     break
 
             if (
-                "Government of Canada. This document is the property of the Government of Canada. It shall not be altered,"
-                in line
+                    "Government of Canada. This document is the property of the Government of Canada. It shall not be altered,"
+                    in line
             ):
                 REPORTNUM_STR = "Evaluation number:"
                 for offset in range(1, 20):
@@ -560,14 +557,14 @@ def search_only_headers_canada(filepath: Path):  # noqa: C901
                             items_found = {}
                             no_match_yet = False
                         line_certid = lines[line_index + offset - 4]
-                        cert_id = line_certid[line_certid.find(REPORTNUM_STR) + len(REPORTNUM_STR) :]
+                        cert_id = line_certid[line_certid.find(REPORTNUM_STR) + len(REPORTNUM_STR):]
                         break
                 if not no_match_yet:
                     break
 
             if (
-                "UNCLASSIFIED / NON CLASSIFIÉ" in line
-                and "COMMON CRITERIA CERTIFICATION REPORT" in lines[line_index + 2]
+                    "UNCLASSIFIED / NON CLASSIFIÉ" in line
+                    and "COMMON CRITERIA CERTIFICATION REPORT" in lines[line_index + 2]
             ):
                 line_certid = lines[line_index + 1]
                 if no_match_yet:
@@ -588,23 +585,6 @@ def search_only_headers_canada(filepath: Path):  # noqa: C901
     return constants.RETURNCODE_OK, items_found
 
 
-def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, int]]]]:
-    try:
-        result = parse_cert_file(filepath, cc_search_rules, -1, constants.LINE_SEPARATOR)
-
-        processed_result = {}
-        top_level_keys = list(result.keys())
-        for key in top_level_keys:
-            processed_result[key] = {key: val for key, val in gen_dict_extract(result[key])}
-
-    except Exception as e:
-        relative_filepath = "/".join(str(filepath).split("/")[-4:])
-        error_msg = f"Failed to parse keywords from: {relative_filepath}; {e}"
-        logger.error(error_msg)
-        return error_msg, None
-    return constants.RETURNCODE_OK, processed_result
-
-
 def search_files(folder: str) -> Iterator[str]:
     for root, _, files in os.walk(folder):
         yield from [os.path.join(root, x) for x in files]
@@ -624,43 +604,50 @@ def save_modified_cert_file(target_file: Union[str, Path], modified_cert_file_te
         write_file.close()
 
 
-def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR):  # noqa: C901
-    whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-        file_name, limit_max_lines, line_separator
-    )
+def extract_keywords(filepath: Path, search_rules) -> Optional[Dict[str, Dict[str, int]]]:
+    """
 
-    items_found_all = {}
-    for rule_group, rules in search_rules.items():
-        if rule_group not in items_found_all:
-            items_found_all[rule_group] = {}
+    """
+    try:
+        return parse_cert_file(filepath, search_rules, -1, constants.LINE_SEPARATOR)
+    except Exception as e:
+        relative_filepath = "/".join(str(filepath).split("/")[-4:])
+        error_msg = f"Failed to parse keywords from: {relative_filepath}; {e}"
+        logger.error(error_msg)
+        return None
 
-        items_found = items_found_all[rule_group]
 
-        for rule in rules:
-            rule_str, rule_and_sep = rule
+def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR):  # noqa: C901
+    """
 
-            for m in re.finditer(rule_and_sep, whole_text):
-                if rule_str not in items_found:
-                    items_found[rule_str] = {}
+    """
+    whole_text, whole_text_with_newlines, was_unicode_decode_error = load_text_file(
+        file_name, limit_max_lines, line_separator
+    )
 
-                match = m.group()
+    def extract(rules):
+        if isinstance(rules, dict):
+            return {k: extract(v) for k, v in rules.items()}
+        elif isinstance(rules, list):
+            matches = [extract(rule) for rule in rules]
+            c = Counter()
+            for match_list in matches:
+                c += Counter(match_list)
+            return dict(c)
+        elif isinstance(rules, re.Pattern):
+            rule = rules
+            matches = []
+            for match in rule.finditer(whole_text):
+                match = match.group()
                 match = normalize_match_string(match)
 
                 match_len = len(match)
                 if match_len > MAX_ALLOWED_MATCH_LENGTH:
-                    logger.warning(f"Excessive match with length of {match_len} detected for rule {rule_str}")
-
-                if match not in items_found[rule_str]:
-                    items_found[rule_str][match] = {}
-                    items_found[rule_str][match][TAG_MATCH_COUNTER] = 0
-                    if APPEND_DETAILED_MATCH_MATCHES:
-                        items_found[rule_str][match][TAG_MATCH_MATCHES] = []
-                items_found[rule_str][match][TAG_MATCH_COUNTER] += 1
-                match_span = m.span()
-                if APPEND_DETAILED_MATCH_MATCHES:
-                    items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1]])
+                    logger.warning(f"Excessive match with length of {match_len} detected for rule {rule.pattern}")
+                matches.append(match)
+            return matches
 
-    return items_found_all
+    return extract(search_rules)
 
 
 def normalize_match_string(match: str) -> str:
@@ -668,36 +655,42 @@ def normalize_match_string(match: str) -> str:
     return "".join(filter(str.isprintable, match))
 
 
-def load_cert_file(
-    file_name: Union[str, Path], limit_max_lines: int = -1, line_separator: str = LINE_SEPARATOR
+def load_text_file(
+        file_name: Union[str, Path], limit_max_lines: int = -1, line_separator: str = LINE_SEPARATOR
 ) -> Tuple[str, str, bool]:
+    """
+    Load the text contents of a file at `file_name`, upto `limit_max_lines` of lines, replace
+    newlines in the text with `line_separator`.
+
+    :param file_name: The file_name to load.
+    :param limit_max_lines: The limit on number of lines to return.
+    :param line_separator: The string to replace newlines with.
+    :return: A tuple of three elements (the text with replaced newlines, the text and a boolean whether a unicode
+             decoding error happened).
+    """
     lines = []
     was_unicode_decode_error = False
     with Path(file_name).open("r", errors=FILE_ERRORS_STRATEGY) as f:
         try:
             lines = f.readlines()
         except UnicodeDecodeError:
-            f.close()
             was_unicode_decode_error = True
-            print("  WARNING: UnicodeDecodeError, opening as utf8")
+            logger.warning("UnicodeDecodeError, opening as utf8")
 
-            with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
-                # coding failure, try line by line
-                line = " "
-                while line:
-                    try:
-                        line = f2.readline()
-                        lines.append(line)
-                    except UnicodeDecodeError:
-                        # ignore error
-                        continue
+    if was_unicode_decode_error:
+        with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
+            # coding failure, try line by line
+            line = " "
+            while line:
+                try:
+                    line = f2.readline()
+                    lines.append(line)
+                except UnicodeDecodeError:
+                    # ignore error
+                    continue
 
     whole_text = ""
     whole_text_with_newlines = ""
-    # we will estimate the line for searched matches
-    # => we need to known how much lines were modified (removal of eoln..)
-    # for removed newline and for any added separator
-    # line_length_compensation = 1 - len(LINE_SEPARATOR)
     lines_included = 0
     for line in lines:
         if limit_max_lines != -1 and lines_included >= limit_max_lines:
@@ -715,15 +708,16 @@ def load_cert_file(
 def load_cert_html_file(file_name: str) -> str:
     with open(file_name, "r", errors=FILE_ERRORS_STRATEGY) as f:
         try:
-            whole_text = f.read()
+            return f.read()
         except UnicodeDecodeError:
-            f.close()
-            with open(file_name, "r", encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
-                try:
-                    whole_text = f2.read()
-                except UnicodeDecodeError:
-                    print("### ERROR: failed to read file {}".format(file_name))
-    return whole_text
+            logger.warning("UnicodeDecodeError, opening as utf8")
+
+    with open(file_name, "r", encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
+        try:
+            return f2.read()
+        except UnicodeDecodeError:
+            logger.error("Failed to read file {}".format(file_name))
+    return ""
 
 
 def gen_dict_extract(dct: Dict, searched_key: Hashable = "count") -> Generator[Any, None, None]:
diff --git a/tests/data/settings_test.yaml b/tests/data/settings_test.yaml
index cff5a9be..5fd4aaca 100644
--- a/tests/data/settings_test.yaml
+++ b/tests/data/settings_test.yaml
@@ -10,9 +10,6 @@ year_difference_between_validations:
   description: During validation we don't connect certificates with validation dates
     difference higher than _this_
   value: 7
-use_text_with_newlines_during_parsing:
-  description: During keyword search, search in text with newlines
-  value: true
 n_threads:
   description: How many threads to use for parallel computations
   value: 8
diff --git a/tests/test_cc_heuristics.py b/tests/test_cc_heuristics.py
index f1fec196..44be386a 100644
--- a/tests/test_cc_heuristics.py
+++ b/tests/test_cc_heuristics.py
@@ -240,19 +240,19 @@ class TestCommonCriteriaHeuristics(TestCase):
     def test_keywords_heuristics(self):
         extracted_keywords: Dict = self.cc_dset["ebd276cca70fd723"].pdf_data.st_keywords
 
-        self.assertTrue("rules_security_level" in extracted_keywords)
-        self.assertEqual(extracted_keywords["rules_security_level"]["EAL3"], 1)
+        self.assertTrue("cc_security_level" in extracted_keywords)
+        self.assertEqual(extracted_keywords["cc_security_level"]["EAL"]["EAL3"], 1)
 
-        self.assertTrue("rules_security_assurance_components" in extracted_keywords)
-        self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_ARC.1"], 1)
-        self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_FSP.3"], 1)
-        self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_TDS.2"], 1)
+        self.assertTrue("cc_sar" in extracted_keywords)
+        self.assertEqual(extracted_keywords["cc_sar"]["ADV"]["ADV_ARC.1"], 1)
+        self.assertEqual(extracted_keywords["cc_sar"]["ADV"]["ADV_FSP.3"], 1)
+        self.assertEqual(extracted_keywords["cc_sar"]["ADV"]["ADV_TDS.2"], 1)
 
-        self.assertTrue("rules_symmetric_crypto" in extracted_keywords)
-        self.assertEqual(extracted_keywords["rules_symmetric_crypto"]["AES"], 2)
+        self.assertTrue("symmetric_crypto" in extracted_keywords)
+        self.assertEqual(extracted_keywords["symmetric_crypto"]["AES_competition"]["AES"]["AES"], 2)
 
-        self.assertTrue("rules_block_cipher_modes" in extracted_keywords)
-        self.assertEqual(extracted_keywords["rules_block_cipher_modes"]["CBC"], 2)
+        self.assertTrue("cipher_mode" in extracted_keywords)
+        self.assertEqual(extracted_keywords["cipher_mode"]["CBC"]["CBC"], 2)
 
     def test_protection_profiles_matching(self):
         artificial_pp: ProtectionProfile = ProtectionProfile(