Merge branch 'fips' of https://github.com/petrs/sec-certs into fips

3 files changed, 41 insertions, 36 deletions

diff --git a/src/cert_rules.py b/src/cert_rules.py
index 800aa825..a3e0dd34 100644
--- a/src/cert_rules.py
+++ b/src/cert_rules.py
@@ -282,6 +282,8 @@ rules_fips_cert = [
     r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2})",
     r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1})"
 ]
+
+#  rule still too "general"
 rules_fips_security_level = [
     r"[lL]evel (\d)"
 ]
@@ -331,26 +333,9 @@ rules['rules_vulnerabilities'] = rules_vulnerabilities
 rules['rules_other'] = rules_other
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-rules['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids
-# rules['rules_vendor'] = rules_vendor
-# rules['rules_cert_id'] = rules_cert_id
-# rules['rules_protection_profiles'] = rules_protection_profiles
-# rules['rules_technical_reports'] = rules_technical_reports
-# rules['rules_device_id'] = rules_device_id
-# rules['rules_os'] = rules_os
-# rules['rules_standard_id'] = rules_standard_id
-# rules['rules_security_level'] = rules_security_level
-rules['rules_security_level'] = rules_fips_security_level
-# rules['rules_security_assurance_components'] = rules_security_assurance_components
-# rules['rules_security_functional_components'] = rules_security_functional_components
-# rules['rules_javacard'] = rules_javacard
-# rules['rules_crypto_algs'] = rules_crypto_algs
-# rules['rules_ecc_curves'] = rules_ecc_curves
-# rules['rules_cplc'] = rules_cplc
-# rules['rules_crypto_engines'] = rules_crypto_engines
-# rules['rules_crypto_libs'] = rules_crypto_libs
-# rules['rules_defenses'] = rules_defenses
-# rules['rules_certification_process'] = rules_certification_process
-# rules['rules_vulnerabilities'] = rules_vulnerabilities
-# rules['rules_other'] = rules_other
-rules['rules_cert_id'] = rules_fips_cert
+#                           For FIPS
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+fips_rules = {}
+fips_rules['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids
+fips_rules['rules_security_level'] = rules_fips_security_level
+fips_rules['rules_cert_id'] = rules_fips_cert
diff --git a/src/extract_certificates.py b/src/extract_certificates.py
index 3dddd2b6..887f06a4 100644
--- a/src/extract_certificates.py
+++ b/src/extract_certificates.py
@@ -9,7 +9,7 @@ import csv
 import string
 
 from analyze_certificates import is_in_dict
-from cert_rules import rules
+from cert_rules import rules, fips_rules
 from enum import Enum
 import matplotlib.pyplot as plt
 
@@ -1128,7 +1128,8 @@ def extract_certificates_keywords(walk_dir, fragments_dir, file_prefix, write_ou
             os.path.splitext(os.path.basename(file_name))[0])[0]
         # parse certificate, return all matches
         all_items_found[file_cert_name], modified_cert_file = parse_cert_file(
-            file_name, rules, -1, should_censure_right_away=should_censure_right_away, fips_items=fips_items)
+            file_name, fips_rules if fips_items else rules, -1, should_censure_right_away=should_censure_right_away,
+            fips_items=fips_items)
 
         # try to establish the certificate id of the current certificate
         # cert_id[file_cert_name] = estimate_cert_id(
@@ -1437,7 +1438,8 @@ def extract_certificates_metadata_html(file_name):
                 items_found['link_security_target_file_name'] = st_file_name
                 download_files_certs.append(
                     (
-                    items_found['link_cert_report'], cert_file_name, items_found['link_security_target'], st_file_name))
+                        items_found['link_cert_report'], cert_file_name, items_found['link_security_target'],
+                        st_file_name))
                 index_next_item += 1
 
                 items_found['maintainance_updates'] = parse_product_updates(
diff --git a/src/fips_certificates.py b/src/fips_certificates.py
index 8c87a5d8..5d7380ee 100644
--- a/src/fips_certificates.py
+++ b/src/fips_certificates.py
@@ -58,6 +58,9 @@ def parse_table(text):
 
 
 def parse_algorithms(text, in_table=False):
+    # print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
+    # print()
+    # print(text)
     items_found = []
     for m in re.finditer(r"(?:#{}\s?|Cert\.?[^. ]*?\s?)(?:[Cc]\s)?(?P<id>\d+)".format('?' if in_table else ''), text):
         items_found.append({'Certificate': m.group()})
@@ -286,13 +289,19 @@ def extract_page_number(txt):
     :param txt: input chunk
     :return: page number
     """
-    print(txt)
+    # Page # of #
     m = re.findall(r"(?P<pattern>(?:[Pp]age) (?P<page_num>\d+)(?: of \d+))", txt)
     if m:
         return m[-1][-1]
+    # Page #
     m = re.findall(r"(?P<pattern>(?:[Pp]age) (?P<page_num>\d+)(?: of \d+)?)", txt)
     if m:
         return m[-1][-1]
+    # # of #
+    m = re.findall(r"(?P<pattern>(?:[Pp]age)? ?(?P<page_num>\d+)(?: of \d+))", txt)
+    if m:
+        return m[-1][-1]
+    # number alone
     m = re.findall(r"(?P<pattern>(?:[Pp]age)? ?(?P<page_num>\d+)(?: of \d+)?)", txt)
     return m[-1][-1] if m else None
 
@@ -312,21 +321,23 @@ def find_tables(txt, file_name, num_pages):
 
     # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer
     print("~" * 20, file_name, '~' * 20)
-    footer_regex = re.compile(r"(?:Table[^\f]*)(?P<first>(\f[ \t\S]+)$)(?P<second>\n^[ \t\S]+?$)?", re.MULTILINE)
+    footer_regex = re.compile(r"(?:Table[^\f]*)(?P<first>^[\S\t ]*$)\n(?P<second>(\f[ \t\S]+)$)(?P<third>\n^[ \t\S]+?$)?",
+                              re.MULTILINE)
 
     # We have 2 groups, one is optional - trying to parse 2 lines (just in case)
     footer1 = [m.group('first') for m in footer_regex.finditer(txt)]
     footer2 = [m.group('second') for m in footer_regex.finditer(txt)]
-    if len(footer2) < len(footer1):
-        footer2 += [''] * (len(footer1) - len(footer2))
+    footer3 = [m.group('third') for m in footer_regex.finditer(txt)]
+
+    # if len(footer2) < len(footer1):
+    #     footer2 += [''] * (len(footer1) - len(footer2))
 
     # zipping them together
-    footer_complete = [m[0] + m[1] for m in zip(footer1, footer2)]
+    footer_complete = [m[0] + m[1] + m[2] for m in zip(footer1, footer2, footer3) if m[0] is not None and m[1] is not None and m[2] is not None]
 
     # removing None and duplicates
     footers = [extract_page_number(x) for x in footer_complete]
-    footers = list(dict.fromkeys([x for x in footers if x is not None and int(x) < num_pages]))
-
+    footers = list(dict.fromkeys([x for x in footers if x is not None and 0 < int(x) < num_pages]))
     print(footers)
     if footers:
         return footers
@@ -340,12 +351,12 @@ def repair_pdf_page_count(file):
 
 def extract_certs_from_tables(list_of_files, html_items):
     global count
-    # list_of_files = ['/home/stan/sec-certs/files/fips/security_policies/2441.pdf.txt']
+    list_of_files = json.loads(open(FIPS_RESULTS_DIR + 'bakup/broken_files.json').read())
     not_decoded = []
     for REDHAT_FILE in list_of_files:
         if '.txt' not in REDHAT_FILE:
             continue
-
+        #
         # if html_items[extract_filename(REDHAT_FILE[:-8])]['tables_done']:
         #     continue
 
@@ -362,12 +373,19 @@ def extract_certs_from_tables(list_of_files, html_items):
             lst = []
             print("~~~~~~~~~~~~~~~", REDHAT_FILE, "~~~~~~~~~~~~~~~~~~~~~~~")
 
-            data = read_pdf(REDHAT_FILE[:-4], pages=tables)
+            try:
+                data = read_pdf(REDHAT_FILE[:-4], pages=tables, silent=True)
+            except Exception:
+                not_decoded.append(REDHAT_FILE)
+                continue
+
             # find columns with cert numbers
             for df in data:
                 for col in range(len(df.columns)):
                     if 'cert' in df.columns[col].lower() or 'algo' in df.columns[col].lower():
                         lst += parse_algorithms(df.iloc[:, col].to_string(index=False), True)
+
+                # Parse again if someone picks not so descriptive column names
                 lst += parse_algorithms(df.to_string(index=False))
             if lst:
                 if 'fips_algorithms' not in html_items[extract_filename(REDHAT_FILE[:-8])]: