Add more regex rulesm, slightly refactor. (#224)

* Add more evaluation facilities to cert rules.

* Add more crypto rules and split them into groups.

* Rename rules_defenses to rules_side_channels.

* Fix tests, use settings_test.

* Fix import order.

* Fix escape warning.

* Fix pandas warning.

* Flake8.

* Add more TEE rules.

* Optimize regex searching.

* Add typehints to cert_rules.

* Fix import order.

7 files changed, 321 insertions, 138 deletions

diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py
index 914e6b3e..7870cd5e 100644
--- a/sec_certs/cert_rules.py
+++ b/sec_certs/cert_rules.py
@@ -245,9 +245,77 @@ rules_vendor = [
     r"(?:G\&D|G\+D|Giesecke\+Devrient|Giesecke \& Devrient)",
     "Philips",
     "Sagem",
+    "Qualcomm",
+    "Broadcom",
+    "Huawei",
 ]
 
-rules_eval_facilities = ["Serma Technologies", "THALES - CEACI"]
+# From: https://www.commoncriteriaportal.org/labs/
+rules_eval_facilities = [
+    "(Serma Technologies|SERMA|Serma Safety & Security)",
+    "(THALES - CEACI|THALES/CNES)",
+    "Riscure",
+    "Bright[sS]ight",
+    "Applus Laboratories",
+    "SGS",
+    "SGS Bright[sS]ight",
+    "(tuvit|TÜViT|TUViT|TÜV Informationstechnik|TUV Informationstechnik)",
+    "CESTI",
+    "DXC Technology",
+    "Teron Labs",
+    "(EWA|EWA-Canada)",
+    "Lightship Security",
+    "AMOSSYS",
+    "(CEA - LETI|CEA/LETI|CEA-LETI)",
+    "OPPIDA",
+    "Trusted Labs",
+    "atsec",
+    "Deutsche Telekom Security",
+    "(Deutsches Forschungszentrum für künstliche Intelligenz|dfki|DFKI)",
+    "MTG AG",
+    "secuvera",
+    "SRC Security Research & Consulting",
+    "Acucert Labs",
+    "Common Criteria Test Laboratory,? ERTL",
+    "Common Criteria Test Laboratory,? ETDC",
+    "CCLab Software Laboratory",
+    "Deeplab",
+    "IMQ/LPS",
+    "LVS Leonardo",
+    "LVS Technis Blu",
+    "ECSEC Laboratory",
+    "Information Technology Security Center",
+    "Acumen Security",
+    "Booz Allen Hamilton",
+    "Gossamer Security",
+    "Leidos",
+    "UL Verification Services",
+    "BEAM Teknoloji",
+    "Certby Lab",
+    "DEKRA Testing and Certification",
+    "STM ITSEF",
+    "TÜBİTAK BİLGEM",
+    "Leidos Common Criteria Testing Laboratory",
+    "Combitech AB",
+    "Intertek",
+    "Clover Technologies",
+    "LAYAKK SEGURIDAD INFORMATICA",
+    "An Security",
+    "T-Systems International",
+    "KISA",
+    "KOIST",
+    "KSEL",
+    "KOSYAS",
+    "KTR",
+    "KTC",
+    "TTA",
+    "Advanced Data Security",
+    "Nemko System Sikkerhet",
+    "Norconsult AS",
+    "Secura",
+    "UL",
+    "BAE Applied Intelligence",
+]
 
 rules_protection_profiles = [
     "BSI-(?:CC[-_]|)PP[-_]*.+?",
@@ -269,6 +337,30 @@ rules_technical_reports = [
 rules_device_id = [
     "G87-.+?",
     "ATMEL AT.+?",
+    "STM32[FGLHW][0-7][0-9]{1,2}[FGKTSCRVZI][468BCDEFGHI][PHUTY][67]",
+    "SLE[0-9]{2}[A-Z]{3}[0-9]{1-4}[A-Z]{1-3}",
+]
+
+rules_tee = [
+    "TEE",
+    "(ARM )?TrustZone",
+    "(ARM )?(Realm Management Extension|Confidential Compute Architecture)",
+    "(Intel )?SGX",
+    "Cloud Link TEE",
+    "iOS Secure Enclave",
+    "iTrustee",
+    "Trusty",
+    "OPTEE",
+    "QTEE",
+    "TEEgris",
+    "T6",
+    "Kinibi",
+    "SW TEE",
+    "WatchTrust",
+    "(AMD )?(PSP|Platform Security Processor)",
+    "(AMD )?(SEV|Secure Encrypted Virtualization)",
+    "(IBM )?(SSC|Secure Service Container)",
+    "(IBM )?(SE|Secure Execution)",
 ]
 
 rules_os = ["STARCOS(?: [0-9\\.]+?|)", "JCOP[ ]*[0-9]"]
@@ -344,7 +436,6 @@ rules_cc_claims = [
     r"OSP\.[\._\-A-Z]+?",  # Organisational Security Policy
 ]
 
-
 rules_javacard = [
     # '(?:Java Card|JavaCard)',
     # '(?:Global Platform|GlobalPlatform)',
@@ -405,43 +496,147 @@ rules_javacard_packages = [
     r"de\.bsi\.[a-z\.]+",
 ]
 
-rules_crypto_algs = [
-    "RSA[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)",
-    "RSASSAPKCS1-[Vv]1_5",
-    "SHA[-]*(?:160|224|256|384|512)",
-    "AES[-]*(?:128|192|256|)",
-    "SHA-1",
-    "MD5",
+rules_symmetric_crypto = [
+    # AES competition
+    "AES-?(?:128|192|256|)",
+    "Rijndael",
+    "Twofish",
+    "Serpent",
+    "MARS",
+    "HPC",
+    "FROG",
+    "CAST-?(?:128|160|192|224|256|5)",
+    "RC[2456]",
+    "CRYPTON",
+    "DEAL",
+    "E2",
+    "LOKI97",
+    "MAGENTA",
+    "SAFER\\+",
+    # DES related
+    "[3T]?DE[SA]",
+    "Lucifer",
+    # djb
+    "ChaCha20",
+    "Poly1305",
+    "Salsa20",
+    # LWC
+    "(ASCON|Ascon)",
+    "Elephant",
+    "GIFT(-COFB)?",
+    "Grain128(-AEAD)?",
+    "ISAP",
+    "Photon-Beetle",
+    "Romulus",
+    "Sparkle",
+    "TinyJambu",
+    "Xoodyak",
+    "Gimli",
+    # Constructions
     "HMAC",
     "HMAC-SHA-(?:160|224|256|384|512)",
-    "(Diffie-Hellman|DH)",
-    "ECDH",
+    "KMAC",
+    # eSTREAM
+    "HC-[0-9]{3}",
+    "Rabbit",
+    "SOSEMANUK",
+    "MICKEY(-128)?",
+    "Trivium",
+    # CAESAR
+    "ACORN",
+    "AEGIS(-128)?",
+    "Deoxys(-2)?",
+    "COLM",
+    # Misc
+    "IDEA",
+    "Blowfish",
+    "Camellia",
+    "CAST",
+    "ARIA",
+    "SM4",
+    "GOST 28147-89",
+    "Skipjack",
+    "(Skinny|SKINNY)",
+    "Kuznyechik",
+]
+
+rules_asymmetric_crypto = [
+    "RSA[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)",
+    "RSASSAPKCS1-[Vv]1_5",
+    "ECDHE?",
     "ECDSA",
     "EdDSA",
-    "[3T]?DES",
     "ECC",
-    "DTRNG",
-    "TRNG",
-    "RN[GD]",
-    "RBG",
-    "PACE",
+    "(Diffie-Hellman|DH|DHE)",
+    "DSA",
+    "BLS",
+    "ECIES",
 ]
 
-rules_block_cipher_modes = [
-    "ECB",
-    "CBC",
-    "CTR",
-    "CFB",
-    "OFB",
-    "GCM",
+rules_pq_crypto = [
+    "Classic[ -]McEliece",
+    "(CRYSTALS-)?(Kyber|KYBER)",
+    "NTRU",
+    "SABER",
+    "(CRYSTALS-)?(Dilithium|DILITHIUM)",
+    "FALCON",
+    "Rainbow",
+    "BIKE",
+    "Frodo(KEM)?",
+    "HQC",
+    "NTRU[ -]Prime",
+    "SIKE",
+    "GeMSS",
+    "Picnic",
+    "SPHINCS\\+",
 ]
 
+rules_hashes = [
+    # SHA-1
+    "SHA-?1",
+    # SHA-2
+    "SHA-?(?:160|224|256|384|512)",
+    # SHA-3
+    "SHA-?3(-[0-9]{3})?",
+    "Keccak",
+    "SHAKE[0-9]{3}" "(Groestl|Grøstl)",
+    "(Blake|BLAKE)[23][sbX]?",
+    "JH",
+    "Skein",
+    # PHC
+    "Argon",
+    "battcrypt",
+    "Catena",
+    "Lyra2",
+    "Makwa",
+    "POMELO",
+    "Pufferfish",
+    "yescrypt",
+    # Misc
+    "MD[3-6]",
+    "RIPEMD(-?[0-9]{3})?",
+    "Streebog",
+    "Whirpool",
+    # Password hashing (non PHC)
+    "bcrypt",
+    "scrypt",
+    "PBKDF[12]?",
+]
+
+rules_crypto_schemes = ["PACE", "MAC", "KEM", "(KEX|Key [eE]xchange)", "PKE", "(TLS|SSL)"]
+
+rules_randomness = ["DUAL_EC_DRBG", "DTRNG", "[PT]RNG", "DRBG", "RN[GD]", "RBG"]
+
+rules_block_cipher_modes = ["ECB", "CBC", "CTR", "CFB", "OFB", "GCM", "SIV", "XTR", "CCM", "LRW", "XEX", "XTS"]
+
 rules_ecc_curves = [
     "(?:Curve |curve |)P-(192|224|256|384|521)",
     "(?:brainpool|BRAINPOOL)P[0-9]{3}[rkt][12]",
     "(?:secp|sect|SECP|SECT)[0-9]+?[rk][12]",
     "(?:ansit|ansip|ANSIP|ANSIT)[0-9]+?[rk][12]",
     "(?:anssi|ANSSI)[ ]*FRP[0-9]+?v1",
+    "(NIST)? ?[PBK]-[0-9]{3}",
+    "numsp[0-9]{3}[td]1",
     "prime[0-9]{3}v[123]",
     "c2[pto]nb[0-9]{3}[vw][123]",
     "FRP256v1",
@@ -449,8 +644,10 @@ rules_ecc_curves = [
     "Ed(25519|448)",
     "ssc-(160|192|224|256|288|320|384|512)",
     "Tweedle(dee|dum)",
+    "(Pallas|Vesta)",
     "JubJub",
     "BLS(12|24)-[0-9]{3}",
+    "bn[0-9]{3}",
 ]
 
 rules_cplc = [
@@ -497,7 +694,7 @@ rules_crypto_libs = [
 
 rules_IC_data_groups = [r"EF\.DG[1-9][0-6]?", r"EF\.COM", r"EF\.CardAccess", r"EF\.SOD", r"EF\.ChipSecurity"]
 
-rules_defenses = [
+rules_side_channels = [
     "[Mm]alfunction",
     "Leak-Inherent",
     "[Pp]hysical [Pp]robing",
@@ -506,10 +703,23 @@ rules_defenses = [
     "SPA",
     "DPA",
     "DFA",
+    "SIFA",
     "[Ff]+ault [iI]nduction",
     "[Ff]+ault [iI]njection",
     "ROCA",
     "[tT]iming [aA]ttacks?",
+    "[Tt]emplate [aA]ttacks?",
+    "[Pp]rofiled [aA]ttacks?",
+    "[Cc]lustering [aA]ttacks?",
+    "[Dd]eep[ -][lL]earning",
+    "[Cc]old [bB]oot",
+    "[Rr]owhammer",
+    "[Rr]everse [eE]ngineering",
+    "[Ll]attice [aA]ttacks?",
+    "[Oo]racle [aA]ttacks?",
+    "[Bb]leichenbacher [aA]ttacks?",
+    "[Bb]ellcore [aA]ttacks?",
+    "(t-test|TVLA)",
 ]
 
 rules_certification_process = [
@@ -529,6 +739,7 @@ rules_other = [
     "library",
     # 'http[s]*://.+?/'
 ]
+
 rules_fips_remove_algorithm_ids = [
     # --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) ---
     # + added (and #id) everywhere
@@ -660,57 +871,59 @@ rules_fips_htmls = [
 ]
 
 
+def add_rules(rule_dict, group_name, rules, add_sep=True):
+    rule_list = [(rule, re.compile(rule + REGEXEC_SEP if add_sep else rule)) for rule in rules]
+    rule_dict[group_name] = rule_list
+
+
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #                            Common rules
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-common_rules = {}
-common_rules["rules_os"] = rules_os
-common_rules["rules_standard_id"] = rules_standard_id
-common_rules["rules_security_level"] = rules_security_level
-common_rules["rules_security_assurance_components"] = rules_security_assurance_components
-common_rules["rules_security_functional_components"] = rules_security_functional_components
-common_rules["rules_cc_claims"] = rules_cc_claims
-common_rules["rules_javacard"] = rules_javacard
-common_rules["rules_javacard_api_consts"] = rules_javacard_api_consts
-common_rules["rules_javacard_packages"] = rules_javacard_packages
-common_rules["rules_crypto_algs"] = rules_crypto_algs
-common_rules["rules_block_cipher_modes"] = rules_block_cipher_modes
-common_rules["rules_ecc_curves"] = rules_ecc_curves
-common_rules["rules_cplc"] = rules_cplc
-common_rules["rules_crypto_engines"] = rules_crypto_engines
-common_rules["rules_crypto_libs"] = rules_crypto_libs
-common_rules["rules_IC_data_groups"] = rules_IC_data_groups
-common_rules["rules_defenses"] = rules_defenses
-common_rules["rules_certification_process"] = rules_certification_process
-common_rules["rules_vulnerabilities"] = rules_vulnerabilities
-common_rules["rules_other"] = rules_other
-
+common_rules: Dict[str, List[Tuple[str, Pattern]]] = {}
+add_rules(common_rules, "rules_os", rules_os)
+add_rules(common_rules, "rules_standard_id", rules_standard_id)
+add_rules(common_rules, "rules_security_level", rules_security_level)
+add_rules(common_rules, "rules_security_assurance_components", rules_security_assurance_components)
+add_rules(common_rules, "rules_security_functional_components", rules_security_functional_components)
+add_rules(common_rules, "rules_cc_claims", rules_cc_claims)
+add_rules(common_rules, "rules_javacard", rules_javacard)
+add_rules(common_rules, "rules_javacard_api_consts", rules_javacard_api_consts)
+add_rules(common_rules, "rules_javacard_packages", rules_javacard_packages)
+add_rules(common_rules, "rules_symmetric_crypto", rules_symmetric_crypto)
+add_rules(common_rules, "rules_asymmetric_crypto", rules_asymmetric_crypto)
+add_rules(common_rules, "rules_pq_crypto", rules_pq_crypto)
+add_rules(common_rules, "rules_hashes", rules_hashes)
+add_rules(common_rules, "rules_crypto_schemes", rules_crypto_schemes)
+add_rules(common_rules, "rules_randomness", rules_randomness)
+add_rules(common_rules, "rules_block_cipher_modes", rules_block_cipher_modes)
+add_rules(common_rules, "rules_ecc_curves", rules_ecc_curves)
+add_rules(common_rules, "rules_cplc", rules_cplc)
+add_rules(common_rules, "rules_tee", rules_tee)
+add_rules(common_rules, "rules_crypto_engines", rules_crypto_engines)
+add_rules(common_rules, "rules_crypto_libs", rules_crypto_libs)
+add_rules(common_rules, "rules_IC_data_groups", rules_IC_data_groups)
+add_rules(common_rules, "rules_side_channels", rules_side_channels)
+add_rules(common_rules, "rules_certification_process", rules_certification_process)
+add_rules(common_rules, "rules_vulnerabilities", rules_vulnerabilities)
+add_rules(common_rules, "rules_other", rules_other)
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #                               For CC
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-# rules_security_target_class
-rules = {}
-rules["rules_vendor"] = rules_vendor
-rules["rules_cert_id"] = rules_cert_id
-rules["rules_protection_profiles"] = rules_protection_profiles
-rules["rules_technical_reports"] = rules_technical_reports
-rules["rules_device_id"] = rules_device_id
-rules.update(common_rules)
+cc_rules: Dict[str, List[Tuple[str, Pattern]]] = {}
+add_rules(common_rules, "rules_vendor", rules_vendor)
+add_rules(common_rules, "rules_cert_id", rules_cert_id)
+add_rules(common_rules, "rules_protection_profiles", rules_protection_profiles)
+add_rules(common_rules, "rules_technical_reports", rules_technical_reports)
+add_rules(common_rules, "rules_device_id", rules_device_id)
+cc_rules.update(common_rules)
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #                            For FIPS
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-fips_rules_base: Dict[str, List[str]] = {}
-fips_rules_base["rules_fips_algorithms"] = rules_fips_remove_algorithm_ids
-fips_rules_base["rules_to_remove"] = rules_fips_to_remove
-fips_rules_base["rules_security_level"] = rules_fips_security_level
-fips_rules_base["rules_cert_id"] = rules_fips_cert
-fips_common_rules = copy.deepcopy(common_rules)  # make separate copy not to process cc rules by fips's re.compile
-
-fips_rules: Dict[str, List[Pattern[str]]] = {}
-
-for rule in fips_rules_base:
-    fips_rules[rule] = []
-    for current_rule in range(len(fips_rules_base[rule])):
-        fips_rules[rule].append(re.compile(fips_rules_base[rule][current_rule]))
+fips_rules: Dict[str, List[Tuple[str, Pattern]]] = {}
+add_rules(fips_rules, "rules_fips_algorithms", rules_fips_remove_algorithm_ids, add_sep=False)
+add_rules(fips_rules, "rules_to_remove", rules_fips_to_remove, add_sep=False)
+add_rules(fips_rules, "rules_security_level", rules_fips_security_level, add_sep=False)
+add_rules(fips_rules, "rules_cert_id", rules_fips_cert, add_sep=False)
+fips_common_rules = copy.deepcopy(common_rules)
diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index c2434622..e554df0b 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -59,4 +59,5 @@ TAG_KEYWORDS = "keywords"
 FILE_ERRORS_STRATEGY = "surrogateescape"
 STOP_ON_UNEXPECTED_NUMS = False
 APPEND_DETAILED_MATCH_MATCHES = False
+MAX_ALLOWED_MATCH_LENGTH = 300
 LINE_SEPARATOR = " "
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index 8ea091db..d7aa2a00 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -437,7 +437,7 @@ class CCDataset(Dataset[CommonCriteriaCert], ComplexSerializableType):
         ]
 
         # TODO: Now skipping bad lines, smarter heuristics to be built for dumb files
-        df = pd.read_csv(file, engine="python", encoding="windows-1252", error_bad_lines=False)
+        df = pd.read_csv(file, engine="python", encoding="windows-1252", on_bad_lines="skip")
         df = df.rename(columns={x: y for (x, y) in zip(list(df.columns), csv_header)})
 
         df["is_maintenance"] = ~df.maintenance_title.isnull()
diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py
index c56c21c2..171fb49b 100644
--- a/sec_certs/helpers.py
+++ b/sec_certs/helpers.py
@@ -25,12 +25,13 @@ from tqdm import tqdm as tqdm_original
 
 import sec_certs.constants as constants
 from sec_certs.cert_rules import REGEXEC_SEP
-from sec_certs.cert_rules import rules as cc_search_rules
+from sec_certs.cert_rules import cc_rules as cc_search_rules
 from sec_certs.config.configuration import config
 from sec_certs.constants import (
     APPEND_DETAILED_MATCH_MATCHES,
     FILE_ERRORS_STRATEGY,
     LINE_SEPARATOR,
+    MAX_ALLOWED_MATCH_LENGTH,
     TAG_MATCH_COUNTER,
     TAG_MATCH_MATCHES,
 )
@@ -855,7 +856,7 @@ def search_only_headers_canada(filepath: Path):  # noqa: C901
 
 def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, int]]]]:
     try:
-        result = parse_cert_file(filepath, cc_search_rules, -1, constants.LINE_SEPARATOR)[0]
+        result = parse_cert_file(filepath, cc_search_rules, -1, constants.LINE_SEPARATOR)
 
         processed_result = {}
         top_level_keys = list(result.keys())
@@ -921,76 +922,43 @@ def save_modified_cert_file(target_file: Union[str, Path], modified_cert_file_te
         write_file.close()
 
 
-# TODO: Please, refactor me.
 def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR):  # noqa: C901
     whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
         file_name, limit_max_lines, line_separator
     )
 
-    # apply all rules
     items_found_all = {}
-    for rule_group in search_rules.keys():
+    for rule_group, rules in search_rules.items():
         if rule_group not in items_found_all:
             items_found_all[rule_group] = {}
 
         items_found = items_found_all[rule_group]
 
-        for rule in search_rules[rule_group]:
-            if type(rule) != str:
-                rule_str = rule.pattern
-                rule_and_sep = re.compile(rule.pattern + REGEXEC_SEP)
-            else:
-                rule_str = rule
-                rule_and_sep = rule + REGEXEC_SEP
+        for rule in rules:
+            rule_str, rule_and_sep = rule
 
-            # matches_with_newlines_count = sum(1 for _ in re.finditer(rule_and_sep, whole_text_with_newlines))
-            # matches_without_newlines_count = sum(1 for _ in re.finditer(rule_and_sep, whole_text))
-            # for m in re.finditer(rule_and_sep, whole_text_with_newlines):
             for m in re.finditer(rule_and_sep, whole_text):
-                # insert rule if at least one match for it was found
-                if rule not in items_found:
+                if rule_str not in items_found:
                     items_found[rule_str] = {}
 
                 match = m.group()
                 match = normalize_match_string(match)
 
-                MAX_ALLOWED_MATCH_LENGTH = 300
                 match_len = len(match)
                 if match_len > MAX_ALLOWED_MATCH_LENGTH:
-                    print("WARNING: Excessive match with length of {} detected for rule {}".format(match_len, rule))
+                    logger.warning(f"Excessive match with length of {match_len} detected for rule {rule_str}")
 
                 if match not in items_found[rule_str]:
                     items_found[rule_str][match] = {}
                     items_found[rule_str][match][TAG_MATCH_COUNTER] = 0
                     if APPEND_DETAILED_MATCH_MATCHES:
                         items_found[rule_str][match][TAG_MATCH_MATCHES] = []
-                    # else:
-                    #     items_found[rule_str][match][TAG_MATCH_MATCHES] = ['List of matches positions disabled. Set APPEND_DETAILED_MATCH_MATCHES to True']
-
                 items_found[rule_str][match][TAG_MATCH_COUNTER] += 1
                 match_span = m.span()
-                # estimate line in original text file
-                # line_number = get_line_number(lines, line_length_compensation, match_span[0])
-                # start index, end index, line number
-                # items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number])
                 if APPEND_DETAILED_MATCH_MATCHES:
                     items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1]])
 
-    # highlight all found strings (by xxxxx) from the input text and store the rest
-    all_matches = []
-    for rule_group in items_found_all.keys():
-        items_found = items_found_all[rule_group]
-        for rule in items_found.keys():
-            for match in items_found[rule]:
-                all_matches.append(match)
-
-        # if AES string is removed before AES-128, -128 would be left in text => sort by length first
-        # sort before replacement based on the length of match
-        all_matches.sort(key=len, reverse=True)
-        for match in all_matches:
-            whole_text_with_newlines = whole_text_with_newlines.replace(match, "x" * len(match))
-
-    return items_found_all, (whole_text_with_newlines, was_unicode_decode_error)
+    return items_found_all
 
 
 def normalize_match_string(match: str) -> str:
diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py
index b2680f0c..cd38af6b 100644
--- a/sec_certs/sample/fips.py
+++ b/sec_certs/sample/fips.py
@@ -14,7 +14,7 @@ from tabula import read_pdf
 
 import sec_certs.constants as constants
 from sec_certs import helpers
-from sec_certs.cert_rules import REGEXEC_SEP, fips_common_rules, fips_rules
+from sec_certs.cert_rules import fips_common_rules, fips_rules
 from sec_certs.config.configuration import config
 from sec_certs.constants import LINE_SEPARATOR
 from sec_certs.helpers import fips_dgst, load_cert_file, normalize_match_string, save_modified_cert_file
@@ -723,9 +723,8 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
         match = m.group()
         match = normalize_match_string(match)
 
-        MAX_ALLOWED_MATCH_LENGTH = 300
         match_len = len(match)
-        if match_len > MAX_ALLOWED_MATCH_LENGTH:
+        if match_len > constants.MAX_ALLOWED_MATCH_LENGTH:
             logger.warning("Excessive match with length of {} detected for rule {}".format(match_len, rule))
 
         if match not in items_found[rule_str]:
@@ -746,22 +745,14 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
     ) -> Tuple[Dict[Pattern, Dict], str]:
         # apply all rules
         items_found_all: Dict[Pattern, Dict] = {}
-        for rule_group in search_rules.keys():
+        for rule_group, rules in search_rules.items():
             if rule_group not in items_found_all:
                 items_found_all[rule_group] = {}
 
             items_found = items_found_all[rule_group]
 
-            for rule in search_rules[rule_group]:
-                rule_and_sep: Union[Pattern, str]
-                if type(rule) != str:
-                    rule_str = rule.pattern
-                    rule_and_sep = re.compile(rule.pattern + REGEXEC_SEP)
-                else:
-                    rule_str = rule
-                    rule_and_sep = rule + REGEXEC_SEP
-
-                for m in re.finditer(rule_and_sep, text_to_parse):
+            for rule_str, rule in rules:
+                for m in re.finditer(rule, text_to_parse):
                     FIPSCertificate._process_match(rule, items_found, rule_str, m)
 
         # highlight all found strings (by xxxxx) from the input text and store the rest
@@ -775,18 +766,17 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
         # apply all rules
         items_found_all: Dict = {}
 
-        for rule_group in fips_rules.keys():
+        for rule_group, rules in fips_rules.items():
             if rule_group not in items_found_all:
                 items_found_all[rule_group] = {}
 
             items_found: Dict[str, Dict] = items_found_all[rule_group]
 
-            for rule in fips_rules[rule_group]:
+            for rule_str, rule in rules:
                 for m in rule.finditer(text_to_parse):
-                    # for m in re.finditer(rule, whole_text_with_newlines):
                     # insert rule if at least one match for it was found
-                    if rule.pattern not in items_found:
-                        items_found[rule.pattern] = {}
+                    if rule_str not in items_found:
+                        items_found[rule_str] = {}
 
                     match = m.group()
                     match = normalize_match_string(match)
@@ -794,11 +784,11 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.FIPSHeuris
                     if match == "":
                         continue
 
-                    if match not in items_found[rule.pattern]:
-                        items_found[rule.pattern][match] = {}
-                        items_found[rule.pattern][match][constants.TAG_MATCH_COUNTER] = 0
+                    if match not in items_found[rule_str]:
+                        items_found[rule_str][match] = {}
+                        items_found[rule_str][match][constants.TAG_MATCH_COUNTER] = 0
 
-                    items_found[rule.pattern][match][constants.TAG_MATCH_COUNTER] += 1
+                    items_found[rule_str][match][constants.TAG_MATCH_COUNTER] += 1
 
                     text_to_parse = text_to_parse.replace(match, "x" * len(match))
 
diff --git a/tests/test_cc_heuristics.py b/tests/test_cc_heuristics.py
index d289a4ba..95a2c4fd 100644
--- a/tests/test_cc_heuristics.py
+++ b/tests/test_cc_heuristics.py
@@ -7,6 +7,7 @@ from unittest import TestCase
 import tests.data.test_cc_heuristics
 from sec_certs import constants
 from sec_certs.cert_rules import SARS_IMPLIED_FROM_EAL
+from sec_certs.config.configuration import config
 from sec_certs.dataset import CCDataset, CPEDataset, CVEDataset
 from sec_certs.sample import CPE, CVE, SAR, CommonCriteriaCert, ProtectionProfile
 
@@ -23,6 +24,7 @@ class TestCommonCriteriaHeuristics(TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
+        config.load(cls.data_dir_path.parent / "settings_test.yaml")
         cls.tmp_dir = tempfile.TemporaryDirectory()
         shutil.copytree(cls.data_dir_path, cls.tmp_dir.name, dirs_exist_ok=True)
 
@@ -122,9 +124,9 @@ class TestCommonCriteriaHeuristics(TestCase):
 
     def test_cpe_parsing(self):
         potentially_problematic_cpes = {
-            'cpe:2.3:a:bayashi:dopvstar\::0091:*:*:*:*:*:*:*"': ("bayashi", "dopvstar:", "0091"),  # noqa: W605
-            "cpe:2.3:a:moundlabs:\:\:mound\:\::2.1.6:*:*:*:*:*:*:*": ("moundlabs", "::mound::", "2.1.6"),  # noqa: W605
-            "cpe:2.3:a:lemonldap-ng:lemonldap\:\::*:*:*:*:*:*:*:*": (  # noqa: W605
+            'cpe:2.3:a:bayashi:dopvstar\\::0091:*:*:*:*:*:*:*"': ("bayashi", "dopvstar:", "0091"),
+            "cpe:2.3:a:moundlabs:\\:\\:mound\\:\\::2.1.6:*:*:*:*:*:*:*": ("moundlabs", "::mound::", "2.1.6"),
+            "cpe:2.3:a:lemonldap-ng:lemonldap\\:\\::*:*:*:*:*:*:*:*": (
                 "lemonldap-ng",
                 "lemonldap::",
                 constants.CPE_VERSION_NA,
@@ -239,8 +241,8 @@ class TestCommonCriteriaHeuristics(TestCase):
         self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_FSP.3"], 1)
         self.assertEqual(extracted_keywords["rules_security_assurance_components"]["ADV_TDS.2"], 1)
 
-        self.assertTrue("rules_crypto_algs" in extracted_keywords)
-        self.assertEqual(extracted_keywords["rules_crypto_algs"]["AES"], 2)
+        self.assertTrue("rules_symmetric_crypto" in extracted_keywords)
+        self.assertEqual(extracted_keywords["rules_symmetric_crypto"]["AES"], 2)
 
         self.assertTrue("rules_block_cipher_modes" in extracted_keywords)
         self.assertEqual(extracted_keywords["rules_block_cipher_modes"]["CBC"], 2)
diff --git a/tests/test_cc_oop.py b/tests/test_cc_oop.py
index 479d3b05..d2809fbe 100644
--- a/tests/test_cc_oop.py
+++ b/tests/test_cc_oop.py
@@ -5,15 +5,24 @@ import tempfile
 from datetime import date, datetime
 from pathlib import Path
 from tempfile import NamedTemporaryFile, TemporaryDirectory
+from typing import ClassVar
 from unittest import TestCase
 
 import sec_certs.constants as constants
 import sec_certs.helpers as helpers
+import tests.data.test_cc_oop
+from sec_certs.config.configuration import config
 from sec_certs.dataset import CCDataset
 from sec_certs.sample import CommonCriteriaCert, ProtectionProfile
 
 
 class TestCommonCriteriaOOP(TestCase):
+    data_dir_path: ClassVar[Path] = Path(tests.data.test_cc_oop.__path__[0])
+
+    @classmethod
+    def setUpClass(cls):
+        config.load(cls.data_dir_path.parent / "settings_test.yaml")
+
     def setUp(self):
         self.test_data_dir = Path(__file__).parent / "data" / "test_cc_oop"
         self.crt_one = CommonCriteriaCert(