regexes overdone

4 files changed, 31 insertions, 8 deletions

diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py
index c85f25bf..877556a1 100644
--- a/sec_certs/cert_rules.py
+++ b/sec_certs/cert_rules.py
@@ -413,7 +413,12 @@ rules_fips_remove_algorithm_ids = [
     r"#\d+, ?#\d+",
     r"#?\d+ and #?\d+",
     r"label \(#\d+\)",
-    r"\(#\d\)"
+    r"\(#\d\)",
+    r"IETF[25\s]*RFC[26\s]*#\d+", # #3425
+    r"Bendix Road North #760", # #3325
+    r"5080 Spectrum Drive, #1000E",
+    r"Document # 540-105000-A1",
+    r"Certificate #2287-1 from EMCE Engineering", # ???
 ]
 
 rules_fips_cert = [
@@ -421,10 +426,10 @@ rules_fips_cert = [
     #     r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{3})",
     #     r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{2})",
     #     r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{1})
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{4})",
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{3})",
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2})",
-    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1})"
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{4}[^\d])",
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{3}[^\d])",
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2}[^\d])",
+    r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1}[^\d])"
 ]
 
 #  rule still too "general"
@@ -501,7 +506,7 @@ fips_common_rules = copy.deepcopy(common_rules)  # make separate copy not to pro
 
 for rule in fips_rules:
     for current_rule in range(len(fips_rules[rule])):
-        fips_rules[rule][current_rule] = re.compile(fips_rules[rule][current_rule] + REGEXEC_SEP)
+        fips_rules[rule][current_rule] = re.compile(fips_rules[rule][current_rule])
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #                         JSON Settings
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index 2c8de742..1e8c85e1 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -682,6 +682,11 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
             self.keywords['rules_cert_id'][rule].pop(self.cert_id, None)
 
+    @staticmethod
+    def get_compare(vendor: str):
+        vendor_split = vendor.replace(',', '')\
+            .replace('-', ' ').replace('+', ' ').replace('®', '').split()
+        return vendor_split[0] if len(vendor_split) > 0 else vendor
 
 class CommonCriteriaCert(Certificate, ComplexSerializableType):
     cc_url = 'http://www.commoncriteriaportal.org'
diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index 6d4aade5..f22bd55d 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -1,4 +1,5 @@
 from enum import Enum
+from .cert_rules import configuration
 
 N_THREADS = 8
 RESPONSE_OK = 200
@@ -41,4 +42,6 @@ TAG_PP_SPONSOR = 'pp_sponsor'
 TAG_PP_EDITOR = 'pp_editor'
 TAG_PP_REVIEWER = 'pp_reviewer'
 TAG_KEYWORDS = 'keywords'
-FIPS_NOT_AVAILABLE_CERT_SIZE = 10000
\ No newline at end of file
+FIPS_NOT_AVAILABLE_CERT_SIZE = 10000
+FIPS_SMALLEST_CERT_ID_TO_CONNECT = configuration['smallest_certificate_id_to_connect']['value']
+FIPS_YEAR_DIFFERENCE_BETWEEN_VALIDATION = configuration["year_difference_between_validations"]["value"]
\ No newline at end of file
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 02034474..a8396fc6 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -3,6 +3,7 @@ from datetime import datetime
 import locale
 import logging
 from typing import Dict, List, ClassVar, Collection, Union, Set, Tuple
+from itertools import groupby
 
 import json
 from abc import ABC, abstractmethod
@@ -719,7 +720,8 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
             algs = self.algorithms.certs[cert_candidate]
             for current_alg in algs:
-                if processed_cert.vendor[:3] in current_alg.vendor:
+                if FIPSCertificate.get_compare(processed_cert.vendor) == FIPSCertificate.get_compare(current_alg.vendor):
+                # if processed_cert.vendor[:3] in current_alg.vendor:
                     return False
             return True
 
@@ -848,6 +850,14 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         dset.root_dir = input_path.parent.absolute()
         return dset
 
+    def group_vendors(self):
+        vendors = {}
+        v = set([x.vendor.lower() for x in self.certs.values()])
+        v = sorted(v, key=FIPSCertificate.get_compare)
+        for prefix, a in groupby(v, key=FIPSCertificate.get_compare):
+            vendors[prefix] = list(a)
+        print(json.dumps(vendors, indent=4))
+
 
 class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):