works in parallel

2 files changed, 16 insertions, 20 deletions

diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index 74b20b4e..512d3c94 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -74,7 +74,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
             return cls(Path(dct['sp_path']), Path(dct['html_path']), Path(dct['fragment_path']))
 
         def to_dict(self):
-            return copy.deepcopy(self.__dict__)
+            return self.__dict__
 
         sp_path: Path
         html_path: Path
@@ -246,7 +246,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                 current_text):
             set_items.add(m.group())
 
-        return list(set_items)
+        return [{"Certificate": [x]} for x in set_items]
 
     @staticmethod
     def parse_table(element: Union[Tag, NavigableString]) -> List[Dict]:
@@ -415,9 +415,9 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         return cert
 
     @staticmethod
-    def parse_cert_file(cert: 'FIPSCertificate') -> Optional[Dict]:
+    def parse_cert_file(cert: 'FIPSCertificate') -> Tuple[Optional[Dict], 'FIPSCertificate']:
         if not cert.txt_state:
-            return None
+            return None, cert
 
         _, whole_text_with_newlines, unicode_error = load_cert_file(cert.state.sp_path.with_suffix('.pdf.txt'), -1,
                                                                     LINE_SEPARATOR)
@@ -466,19 +466,17 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                         match, 'x' * len(match))
 
         save_modified_cert_file(cert.state.fragment_path, whole_text_with_newlines, unicode_error)
-        return items_found_all
+        return items_found_all, cert
 
     @staticmethod
     def analyze_tables(cert: 'FIPSCertificate') -> Tuple[bool, 'FIPSCertificate', List]:
         cert_file = cert.state.sp_path
         txt_file = cert_file.with_suffix('.pdf.txt')
-        print(txt_file)
         with open(txt_file, 'r') as f:
             tables = helpers.find_tables(f.read(), txt_file)
 
         # If we find any tables with page numbers, we process them
         lst = []
-        print(tables)
         if tables:
             try:
                 data = read_pdf(cert_file, pages=tables, silent=True)
@@ -502,7 +500,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                 # Parse again if someone picks not so descriptive column names
                 lst += FIPSCertificate.parse_algorithms(df.to_string(index=False))
 
-            lst += {"PLS": "DO I WORK MAKE ME WORK"}
+            lst += [{"PLS": "DO I WORK MAKE ME WORK"}]
 
         return True, cert, lst
 
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 32f04732..6ea28975 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -530,8 +530,8 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                                                         [cert for cert in self.certs.values() if not cert.keywords],
                                                         constants.N_THREADS,
                                                         use_threading=False)
-            for keyword, cert in zip(keywords, self.certs.values()):
-                cert.keywords = keyword
+            for keyword, cert in keywords:
+                self.certs[cert.dgst].keywords = keyword
         else:
             self.keywords = json.loads(
                 open(self.root_dir / 'fips_full_keywords.json').read())
@@ -657,20 +657,17 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         result = cert_processing.process_parallel(FIPSCertificate.analyze_tables,
                                                   [cert for cert in self.certs.values() if
                                                    not cert.tables_done and cert.txt_state],
-                                                  constants.N_THREADS,
+                                                  4,
                                                   use_threading=False)
 
         not_decoded = list(map(lambda tup: tup[1].state.sp_path, filter(lambda tup: tup[0] is False, result)))
         for state, cert, algorithms in result:
-            cert.tables_done = state
-            cert.algorithms += algorithms
+            self.certs[cert.dgst].tables_done = state
+            self.certs[cert.dgst].algorithms += algorithms
 
         return not_decoded
 
     def remove_algorithms_from_extracted_data(self):
-        """
-        Function that removes all found certificate IDs that are matching any IDs labeled as algorithm IDs
-        """
         for cert in self.certs.values():
             cert.remove_algorithms()
 
@@ -679,9 +676,10 @@ class FIPSDataset(Dataset, ComplexSerializableType):
             new_algorithms = []
             for algorithm in certificate.algorithms:
                 if isinstance(algorithm, dict):
-                    new_algorithms.append(algorithm)
+                    if "PLS" not in algorithm:
+                        new_algorithms.append(algorithm)
                 else:
-                    new_algorithms.append({'Certificate': algorithm})
+                    new_algorithms.append({'Certificate': [algorithm]})
             certificate.algorithms = new_algorithms
 
     def validate_results(self):
@@ -717,8 +715,8 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                     cert_id = ''.join(filter(str.isdigit, cert))
 
                     if cert_id == '' or cert_id not in self.certs:
-                        broken_files.add(current_cert)
-                        self.certs[current_cert].file_status = False
+                        broken_files.add(current_cert.dgst)
+                        current_cert.file_status = False
                         break
 
         if broken_files: