WIP: algorithm parsing

2 files changed, 44 insertions, 21 deletions

diff --git a/fips_cli.py b/fips_cli.py
index b63547b6..25a57957 100755
--- a/fips_cli.py
+++ b/fips_cli.py
@@ -199,16 +199,16 @@ def main(
     if 'update' in actions:
         dset.get_certs_from_web(no_download_algorithms=no_download_algs, update=True)
 
-    if "web-scan" in actions:
+    if "web-scan" in actions or "update" in actions:
         dset.web_scan(redo=redo_web_scan)
 
-    if "convert" in actions :
+    if "convert" in actions or "update" in actions:
         dset.convert_all_pdfs()
 
-    if "pdf-scan" in actions:
+    if "pdf-scan" in actions or "update" in actions:
         dset.pdf_scan(redo=redo_keyword_scan)
 
-    if "table-search" in actions:
+    if "table-search" in actions or "update" in actions:
         if not higher_precision_results:
             print(
                 "Info: You are using table search without higher precision results. It is advised to use the switch in the next run."
diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py
index 25086518..89b4e4df 100644
--- a/sec_certs/dataset/fips_algorithm.py
+++ b/sec_certs/dataset/fips_algorithm.py
@@ -27,43 +27,66 @@ class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
             soup = BeautifulSoup(alg_file.read(), 'html.parser')
             num_pages = soup.select('span[data-total-pages]')[0].attrs
 
-        for i in range(1, int(num_pages['data-total-pages'])):
+        for i in range(2, int(num_pages['data-total-pages']) + 1):
             if not (self.root_dir / f'page{i}.html').exists():
                 algs_urls.append(
                     constants.FIPS_ALG_URL + str(i))
                 algs_paths.append(self.root_dir / f"page{i}.html")
 
+        # get the last page, always
+        helpers.download_file(constants.FIPS_ALG_URL + num_pages['data-total-pages'],
+                              self.root_dir / f"page{int(num_pages['data-total-pages'])}.html")
         logging.info(f"downloading {len(algs_urls)} algs html files")
         cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(algs_urls, algs_paths)),
                                          config.n_threads)
 
         self.parse_html()
-        print(self.certs)
+
+    @staticmethod
+    def _extract_algorithm_information(elements, vendor, date, product, validation):
+        for elem in elements:
+            # td > a > (vendor or date)
+            attachments = elem.find_all('a')
+
+            if len(attachments) == 0:
+                vendor = elem.text.strip() if 'vendor-name' in elem['id'] else vendor
+                date = elem.text.strip() if 'validation-date' in elem['id'] else date
+                continue
+
+            for attachment in attachments:
+                product = elem.text.strip() if 'product-name' in attachment['id'] else product
+                validation = elem.text.strip() if 'validation-number' in attachment['id'] else validation
+        return vendor, date, product, validation
 
     def parse_html(self):
         def split_alg(alg_string):
             cert_type = alg_string.rstrip('0123456789')
             cert_id = alg_string[len(cert_type):]
             return cert_type.strip(), cert_id.strip()
-        
-        for f in sec_certs.helpers.search_files(self.root_dir):
+
+        for f in helpers.search_files(self.root_dir):
+            if not f.endswith("html"):
+                continue
+
             with open(f, 'r', encoding='utf-8') as handle:
                 html_soup = BeautifulSoup(handle.read(), 'html.parser')
 
-            table = html_soup.find('table', class_='table table-condensed publications-table table-bordered')
-            spans = table.find_all('span')
-            for span in spans:
-                elements = span.find_all('td')
-                vendor, implementation = elements[0].text, elements[1].text
-                elements_sliced = elements[2:]
-                for i in range(0, len(elements_sliced), 2):
-                    alg_type, alg_id = split_alg(elements_sliced[i].text.strip())
-                    validation_date = elements_sliced[i + 1].text.strip()
-                    fips_alg = FIPSCertificate.Algorithm(alg_id, vendor, implementation, alg_type, validation_date)
-                    if alg_id not in self.certs:
-                        self.certs[alg_id] = []
-                    self.certs[alg_id].append(fips_alg)
+            table = html_soup.find(
+                'table', class_='table table-condensed publications-table table-bordered')
+            tbody_contents = table.find('tbody').find_all('tr')
+            vendor = product = validation = date = ""
+            for tr in tbody_contents:
+                elements = tr.find_all('td')
+                vendor, date, product, validation = FIPSAlgorithmDataset._extract_algorithm_information(
+                    elements, vendor, date, product, validation
+                )
 
+                alg_type, alg_id = split_alg(validation)
+                fips_alg = FIPSCertificate.Algorithm(
+                    alg_id, vendor, product, alg_type, date)
+                if alg_id not in self.certs:
+                    self.certs[alg_id] = []
+                self.certs[alg_id].append(fips_alg)
     def convert_all_pdfs(self):
         raise NotImplementedError('Not meant to be implemented')