keywords work, tables dont

2 files changed, 62 insertions, 51 deletions

diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index c732527a..74b20b4e 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -406,7 +406,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
     def convert_pdf_file(tup: Tuple['FIPSCertificate', Path, Path]) -> 'FIPSCertificate':
         cert, pdf_path, txt_path = tup
         if not cert.txt_state:
-            exit_code = helpers.convert_pdf_file(pdf_path, txt_path, ['-layout'])
+            exit_code = helpers.convert_pdf_file(pdf_path, txt_path, ['-raw'])
             if exit_code != constants.RETURNCODE_OK:
                 logger.error(f'Cert dgst: {cert.dgst} failed to convert security policy pdf->txt')
                 cert.txt_state = False
@@ -415,9 +415,9 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         return cert
 
     @staticmethod
-    def parse_cert_file(cert: 'FIPSCertificate'):
+    def parse_cert_file(cert: 'FIPSCertificate') -> Optional[Dict]:
         if not cert.txt_state:
-            return
+            return None
 
         _, whole_text_with_newlines, unicode_error = load_cert_file(cert.state.sp_path.with_suffix('.pdf.txt'), -1,
                                                                     LINE_SEPARATOR)
@@ -466,28 +466,31 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                         match, 'x' * len(match))
 
         save_modified_cert_file(cert.state.fragment_path, whole_text_with_newlines, unicode_error)
-        cert.keywords = items_found_all
+        return items_found_all
 
     @staticmethod
-    def analyze_tables(cert: 'FIPSCertificate') -> Tuple[bool, Path]:
+    def analyze_tables(cert: 'FIPSCertificate') -> Tuple[bool, 'FIPSCertificate', List]:
         cert_file = cert.state.sp_path
         txt_file = cert_file.with_suffix('.pdf.txt')
+        print(txt_file)
         with open(txt_file, 'r') as f:
             tables = helpers.find_tables(f.read(), txt_file)
 
         # If we find any tables with page numbers, we process them
+        lst = []
+        print(tables)
         if tables:
-            lst = []
             try:
-                data = read_pdf(cert_file,
-                                pages=tables, silent=True)
-            except Exception:
+                data = read_pdf(cert_file, pages=tables, silent=True)
+            except Exception as e:
                 try:
+                    logger.error(e)
                     helpers.repair_pdf(cert_file)
                     data = read_pdf(cert_file, pages=tables, silent=True)
 
-                except Exception:
-                    return False, cert_file
+                except Exception as ex:
+                    logger.error(ex)
+                    return False, cert, lst
 
             # find columns with cert numbers
             for df in data:
@@ -501,11 +504,30 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
 
             lst += {"PLS": "DO I WORK MAKE ME WORK"}
 
-            if lst:
-                cert.algorithms += lst
+        return True, cert, lst
+
+    def remove_algorithms(self):
+        self.file_status = True
+        if not self.keywords:
+            return
+
+        if self.mentioned_certs:
+            for item in self.mentioned_certs:
+                self.keywords['rules_cert_id'].update(item)
+
+        for rule in self.keywords['rules_cert_id']:
+            to_pop = set()
+            rr = re.compile(rule)
+            for cert in self.keywords['rules_cert_id'][rule]:
+                for alg in self.keywords['rules_fips_algorithms']:
+                    for found in self.keywords['rules_fips_algorithms'][alg]:
+                        if rr.search(found) and rr.search(cert) and rr.search(found).group('id') == rr.search(
+                                cert).group('id'):
+                            to_pop.add(cert)
+            for r in to_pop:
+                self.keywords['rules_cert_id'][rule].pop(r, None)
 
-        cert.tables_done = True
-        return True, cert_file
+            self.keywords['rules_cert_id'][rule].pop(self.cert_id, None)
 
 
 class CommonCriteriaCert(Certificate, ComplexSerializableType):
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 2dd07233..32f04732 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -526,10 +526,12 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         self.fragments_dir.mkdir(parents=True, exist_ok=True)
         if self.new_files > 0 or not (self.root_dir / 'fips_full_keywords.json').exists():
 
-            cert_processing.process_parallel(FIPSCertificate.parse_cert_file,
-                                             [cert for cert in self.certs.values() if not cert.keywords],
-                                             constants.N_THREADS,
-                                             use_threading=False)
+            keywords = cert_processing.process_parallel(FIPSCertificate.parse_cert_file,
+                                                        [cert for cert in self.certs.values() if not cert.keywords],
+                                                        constants.N_THREADS,
+                                                        use_threading=False)
+            for keyword, cert in zip(keywords, self.certs.values()):
+                cert.keywords = keyword
         else:
             self.keywords = json.loads(
                 open(self.root_dir / 'fips_full_keywords.json').read())
@@ -587,13 +589,14 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         logger.info('Converting FIPS certificate reports to .txt')
         tuples = [
             (cert, self.policies_dir / f'{cert.cert_id}.pdf', self.policies_dir / f'{cert.cert_id}.pdf.txt')
-            for cert in self.certs.values() if not cert.txt_state
+            for cert in self.certs.values() if not cert.txt_state and (self.policies_dir / f'{cert.cert_id}.pdf').exists()
         ]
         cert_processing.process_parallel(FIPSCertificate.convert_pdf_file, tuples, constants.N_THREADS)
 
     def get_certs_from_web(self):
-        def download_html_pages() -> Tuple[int, int]:
-            # self.download_all_pdfs()
+# there was a Tuple[int, int] return - why?
+        def download_html_pages():
+            self.download_all_pdfs()
             self.download_all_htmls()
             self.download_all_algs()
 
@@ -651,39 +654,25 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         Function that extracts algorithm IDs from tables in security policies files.
         :return: list of files that couldn't have been decoded
         """
-        not_decoded = cert_processing.process_parallel(FIPSCertificate.analyze_tables,
-                                                       [cert for cert in self.certs.values() if
-                                                        not cert.tables_done and cert.txt_state],
-                                                       constants.N_THREADS,
-                                                       use_threading=False)
-        return list(map(lambda tup: tup[1], filter(lambda tup: tup[0] is False, not_decoded)))
+        result = cert_processing.process_parallel(FIPSCertificate.analyze_tables,
+                                                  [cert for cert in self.certs.values() if
+                                                   not cert.tables_done and cert.txt_state],
+                                                  constants.N_THREADS,
+                                                  use_threading=False)
+
+        not_decoded = list(map(lambda tup: tup[1].state.sp_path, filter(lambda tup: tup[0] is False, result)))
+        for state, cert, algorithms in result:
+            cert.tables_done = state
+            cert.algorithms += algorithms
+
+        return not_decoded
 
     def remove_algorithms_from_extracted_data(self):
         """
         Function that removes all found certificate IDs that are matching any IDs labeled as algorithm IDs
         """
-        for file_name in self.keywords:
-            self.keywords[file_name]['file_status'] = True
-            self.certs[file_name].file_status = True
-            if self.certs[file_name].mentioned_certs:
-                for item in self.certs[file_name].mentioned_certs:
-                    self.keywords[file_name]['rules_cert_id'].update(item)
-
-            for rule in self.keywords[file_name]['rules_cert_id']:
-                to_pop = set()
-                rr = re.compile(rule)
-                for cert in self.keywords[file_name]['rules_cert_id'][rule]:
-                    for alg in self.keywords[file_name]['rules_fips_algorithms']:
-                        for found in self.keywords[file_name]['rules_fips_algorithms'][alg]:
-                            if rr.search(found) and rr.search(cert) and rr.search(found).group('id') == rr.search(
-                                    cert).group('id'):
-                                to_pop.add(cert)
-                for r in to_pop:
-                    self.keywords[file_name]['rules_cert_id'][rule].pop(
-                        r, None)
-
-                self.keywords[file_name]['rules_cert_id'][rule].pop(
-                    self.certs[file_name].cert_id, None)
+        for cert in self.certs.values():
+            cert.remove_algorithms()
 
     def unify_algorithms(self):
         for certificate in self.certs.values():
@@ -740,7 +729,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
 
         for current_cert in self.certs.values():
             current_cert.connections = []
-            if not current_cert.file_status:
+            if not current_cert.file_status or not current_cert.keywords:
                 continue
             if current_cert.keywords['rules_cert_id'] == {}:
                 continue