Fixed changes

4 files changed, 33 insertions, 16 deletions

diff --git a/fips_oop_demo.py b/fips_oop_demo.py
index f2558c24..3d0bb8a6 100644
--- a/fips_oop_demo.py
+++ b/fips_oop_demo.py
@@ -1,4 +1,4 @@
-from sec_certs.dataset import FIPSDataset, AlgorithmDataset
+from sec_certs.dataset import FIPSDataset, FIPSAlgorithmDataset
 from pathlib import Path
 from datetime import datetime
 import logging
@@ -42,7 +42,7 @@ def main():
     logging.info(f"Done. Files not decoded: {not_decoded_files}")
 
     logging.info("Parsing algorithms")
-    aset = AlgorithmDataset({}, Path('fips_dataset/web/algorithms'), 'algorithms', 'sample algs')
+    aset = FIPSAlgorithmDataset({}, Path('fips_dataset/web/algorithms'), 'algorithms', 'sample algs')
     aset.parse_html()
 
     dset.algorithms = aset
diff --git a/sec_certs/cert_processing.py b/sec_certs/cert_processing.py
index 2e4fb3ee..739bfa9f 100644
--- a/sec_certs/cert_processing.py
+++ b/sec_certs/cert_processing.py
@@ -11,7 +11,7 @@ def process_parallel(func: Callable, items: Iterable, max_workers: int, callback
     else:
         pool = Pool(max_workers)
 
-    results = [pool.apply_async(func, (*i, ), callback=callback) for i in items]
+    results = [pool.apply_async(func, (i, ), callback=callback) for i in items]
 
     if progress_bar is True:
         bar = tqdm(total=len(results))
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index 5682fb58..7da5451e 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -10,7 +10,7 @@ import requests
 
 from abc import ABC, abstractmethod
 from bs4 import Tag, BeautifulSoup, NavigableString
-from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type
+from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type, Tuple
 
 from sec_certs import helpers, extract_certificates, dataset
 from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
@@ -88,7 +88,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                  tables: bool,
                  file_status: Optional[bool],
                  connections: List,
-                 txt_state: bool=True):
+                 txt_state: bool=False):
         super().__init__()
         self.cert_id = cert_id
 
@@ -131,6 +131,20 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
         return self.cert_id
 
     @staticmethod
+    def download_security_policy(cert: Tuple[str, Path]) -> None:
+        exit_code = helpers.download_file(*cert)
+        if exit_code != requests.codes.ok:
+            logger.error(f'Failed to download security policy from {cert[0]}, code: {exit_code}')
+        return cert
+
+    @staticmethod
+    def download_html_page(cert: Tuple[str, Path]) -> None:
+        exit_code = helpers.download_file(*cert)
+        if exit_code != requests.codes.ok:
+            logger.error(f'Failed to download html page from {cert[0]}, code: {exit_code}')
+        return cert
+
+    @staticmethod
     def extract_filename(file: str) -> str:
         """
         Extracts filename from path
@@ -337,11 +351,12 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                                [])
 
     @staticmethod
-    def convert_pdf_file(cert: 'FipsCertificate', ds: 'FIPSDataset') -> 'FIPSCertificate':
+    def convert_pdf_file(tup: Tuple['FIPSCertificate', Path, Path]) -> 'FIPSCertificate':
+        cert, pdf_path, txt_path = tup
         if not cert.txt_state:
-            exit_code = helpers.convert_pdf_file(ds.policies_dir / f'{cert.cert_id}.pdf', ds.policies_dir / f'{cert.cert_id}.pdf.txt', ['-layout'])
+            exit_code = helpers.convert_pdf_file(pdf_path, txt_path, ['-layout'])
             if exit_code != constants.RETURNCODE_OK:
-                logger.error(f'Cert dgst: {cert.dgst} failed to convert security target pdf->txt')
+                logger.error(f'Cert dgst: {cert.dgst} failed to convert security policy pdf->txt')
                 cert.txt_state = False
         return cert
 
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index b35adf2f..9a9d4d68 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -546,7 +546,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                 sp_paths.append(self.policies_dir / f"{cert_id}.pdf")
 
         logging.info(f"downloading {len(sp_urls)} module pdf files")
-        Dataset._download_parallel(sp_urls, sp_paths)
+        cert_processing.process_parallel(FIPSCertificate.download_security_policy, list(zip(sp_urls, sp_paths)), constants.N_THREADS)
         self.new_files += len(sp_urls)
 
     def download_all_htmls(self):
@@ -559,7 +559,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                 html_paths.append(self.web_dir / f"{cert_id}.html")
 
         logging.info(f"downloading {len(html_urls)} module html files")
-        Dataset._download_parallel(html_urls, html_paths)
+        cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(html_urls, html_paths)), constants.N_THREADS)
         self.new_files += len(html_urls)
 
     def download_all_algs(self):
@@ -572,17 +572,19 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                 algs_paths.append(self.algs_dir / f"page{i}.html")
 
         logging.info(f"downloading {len(algs_urls)} algs html files")
-        Dataset._download_parallel(algs_urls, algs_paths)
+        cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(algs_urls, algs_paths)), constants.N_THREADS)
         self.new_files += len(algs_urls)
     
 
     def convert_all_pdfs(self):
-        logger.info('Converting CC certificate reports to .txt')
-        for cert in self.certs.values():
-            FIPSCertificate.convert_pdf_file(cert, self)
+        logger.info('Converting FIPS certificate reports to .txt')
+        tuples = [
+            (cert, self.policies_dir / f'{cert.cert_id}.pdf', self.policies_dir / f'{cert.cert_id}.pdf.txt')
+            for cert in self.certs.values() if not cert.txt_state
+        ]
+        cert_processing.process_parallel(FIPSCertificate.convert_pdf_file, tuples, constants.N_THREADS)
 
 
-    # TODO figure out whether the name of this method shuold not be "get_certs", because we don't download every time
     def get_certs_from_web(self):
         def download_html_pages() -> Tuple[int, int]:
             self.download_all_pdfs()
@@ -876,7 +878,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
         return dset
 
 
-class AlgorithmDataset(Dataset, ComplexSerializableType):
+class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
 
     def get_certs_from_web(self):
         pass