fips: oop working (#15)

* FIPS Certificate constructor

* Parsing of single certificate done

* Downloading files

* Create dataset of objects

* demo + dumping + keyword extraction

* wait for approval + everything changed

* non trivial amount of changes to make this work

* modified tests to work with new fips html files

* merge comments resolved (first commit)

* merge comments resolved (second commit)

* realizing a better idea is to leave it alone

* commit #3

Co-authored-by: Stanislav Boboň <xbobon@fi.muni.cz>

11 files changed, 743 insertions, 338 deletions

diff --git a/fips_oop_demo.py b/fips_oop_demo.py
new file mode 100644
index 00000000..6f556394
--- /dev/null
+++ b/fips_oop_demo.py
@@ -0,0 +1,45 @@
+from sec_certs.dataset import FIPSDataset
+from pathlib import Path
+from datetime import datetime
+import logging
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    start = datetime.now()
+
+    # Create empty dataset
+    dset = FIPSDataset({}, Path('./fips_dataset'), 'sample_dataset', 'sample dataset description')
+
+    # Load metadata for certificates from CSV and HTML sources
+    dset.get_certs_from_web()
+    logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.')
+    # Dump dataset into JSON
+
+    dset.dump_to_json()
+    logging.info(f'Dataset saved to {dset.root_dir}/fips_full_dataset.json')
+
+    logging.info("Extracting keywords now.")
+
+    dset.extract_keywords()
+
+    logging.info(f'Finished extracting certificates for {len(dset.keywords)} items.')
+    logging.info(f'Dumping keywords to {dset.root_dir}/fips_full_keywords.json')
+    dset.dump_keywords()
+
+    logging.info("Searching for tables in pdfs")
+
+    not_decoded_files = dset.extract_certs_from_tables()
+
+    logging.info(f"Done. Files not decoded: {not_decoded_files}")
+
+    logging.info("finalizing results.")
+
+    dset.finalize_results()
+
+    end = datetime.now()
+    logging.info(f'The computation took {(end - start)} seconds.')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index 2f8254bf..2840cc75 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -1,10 +1,14 @@
+import re
 from datetime import datetime, date
 from dataclasses import dataclass
 import logging
-from . import helpers
+from pathlib import Path
+import os
+
+from . import helpers, extract_certificates
 from abc import ABC, abstractmethod
-from bs4 import Tag
-from typing import Union, Optional
+from bs4 import Tag, BeautifulSoup, NavigableString
+from typing import Union, Optional, List, Dict, ClassVar
 
 
 class Certificate(ABC):
@@ -33,19 +37,285 @@ class Certificate(ABC):
     def from_dict(cls, dct: dict) -> 'Certificate':
         raise NotImplementedError('Mot meant to be implemented')
 
-    @abstractmethod
-    def merge(self, other: 'Certificate'):
-        raise NotImplementedError('Not meant to be implemented')
-
 
 class FIPSCertificate(Certificate):
+    FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov'
+    FIPS_MODULE_URL: ClassVar[
+        str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
+
+    def __init__(self, cert_id: str,
+                 module_name: Optional[str],
+                 standard: Optional[str],
+                 status: Optional[str],
+                 date_sunset: Optional[List[str]],
+                 date_validation: Optional[List[str]],
+                 level: Optional[str],
+                 caveat: Optional[str],
+                 exceptions: Optional[List[str]],
+                 module_type: Optional[str],
+                 embodiment: Optional[str],
+                 algorithms: Optional[List[str]],
+                 tested_conf: Optional[List[str]],
+                 description: Optional[str],
+                 mentioned_certs: Optional[List[str]],
+                 vendor: Optional[str],
+                 vendor_www: Optional[str],
+                 lab: Optional[str],
+                 lab_nvlap: Optional[str],
+                 historical_reason: Optional[str],
+                 security_policy_www: Optional[str],
+                 certificate_www: Optional[str],
+                 hw_version: Optional[str],
+                 fw_version: Optional[str],
+                 tables: bool,
+                 file_status: Optional[bool],
+                 connections: List):
+        super().__init__()
+        self.cert_id = cert_id
+
+        self.module_name = module_name
+        self.standard = standard
+        self.status = status
+        self.date_sunset = date_sunset
+        self.date_validation = date_validation
+        self.level = level
+        self.caveat = caveat
+        self.exceptions = exceptions
+        self.type = module_type
+        self.embodiment = embodiment
+
+        self.algorithms = algorithms
+        self.tested_conf = tested_conf
+        self.description = description
+        self.mentioned_certs = mentioned_certs
+        self.vendor = vendor
+        self.vendor_www = vendor_www
+        self.lab = lab
+        self.lab_nvlap = lab_nvlap
+
+        self.historical_reason = historical_reason
+        self.security_policy_www = security_policy_www
+        self.certificate_www = certificate_www
+        self.hw_versions = hw_version
+        self.fw_versions = fw_version
+
+        self.tables_done = tables
+        self.file_status = file_status
+        self.connections = connections
+
+    def __str__(self) -> str:
+        return str(self.cert_id)
+
+    @property
+    def dgst(self) -> str:
+        return self.cert_id
+
     @classmethod
     def from_dict(cls, dct: dict) -> 'FIPSCertificate':
-        return FIPSCertificate()
+        args = tuple(dct.values())
+        return FIPSCertificate(*args)
 
-    @property
-    def dgst(self):
-        return None  # TODO: Implement me
+    @staticmethod
+    def extract_filename(file: str) -> str:
+        """
+        Extracts filename from path
+        @param file: UN*X path
+        :return: filename without last extension
+        """
+        return os.path.splitext(os.path.basename(file))[0]
+
+    @staticmethod
+    def initialize_dictionary() -> Dict:
+        d = {'fips_module_name': None, 'fips_standard': None, 'fips_status': None, 'fips_date_sunset': None,
+             'fips_date_validation': None, 'fips_level': None, 'fips_caveat': None, 'fips_exceptions': None,
+             'fips_type': None, 'fips_embodiment': None, 'fips_tested_conf': None, 'fips_description': None,
+             'fips_vendor': None, 'fips_vendor_www': None, 'fips_lab': None, 'fips_lab_nvlap': None,
+             'fips_historical_reason': None, 'fips_algorithms': [], 'fips_mentioned_certs': [],
+             'fips_tables_done': False, 'fips_security_policy_www': None, 'fips_certificate_www': None,
+             'fips_hw_versions': None, 'fips_fw_versions': None}
+
+        return d
+
+    @staticmethod
+    def parse_caveat(current_text: str) -> List:
+        """
+        Parses content of "Caveat" of FIPS CMVP .html file
+        :param current_text: text of "Caveat"
+        :return: list of all found algorithm IDs
+        """
+        ids_found = []
+        r_key = r"(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)(?P<id>\d+)"
+        for m in re.finditer(r_key, current_text):
+            if r_key in ids_found and m.group() in ids_found[0]:
+                ids_found[0][m.group()]['count'] += 1
+            else:
+                ids_found.append(
+                    {r"(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)(?P<id>\d+?})": {m.group(): {'count': 1}}})
+
+        return ids_found
+
+    @staticmethod
+    def parse_algorithms(current_text: str, in_pdf: bool = False) -> List:
+        """
+        Parses table of FIPS (non) allowed algorithms
+        :param current_text: Contents of the table
+        :param in_pdf: Specifies whether the table was found in a PDF security policies file
+        :return: list of all found algorithm IDs
+        """
+        set_items = set()
+        for m in re.finditer(rf"(?:#{'?' if in_pdf else 'C?'}\s?|Cert\.?[^. ]*?\s?)(?:[Cc]\s)?(?P<id>\d+)",
+                             current_text):
+            set_items.add(m.group())
+
+        return list(set_items)
+
+    @staticmethod
+    def parse_table(element: Union[Tag, NavigableString]) -> List[Dict]:
+        """
+        Parses content of <table> tags in FIPS .html CMVP page
+        :param element: text in <table> tags
+        :return: list of all found algorithm IDs
+        """
+        found_items = []
+        trs = element.find_all('tr')
+        for tr in trs:
+            tds = tr.find_all('td')
+            found_items.append({'Name': tds[0].text, 'Certificate': parse_algorithms(tds[1].text)})
+
+        return found_items
+
+    @staticmethod
+    def parse_html_main(current_div: Tag, html_items_found: Dict, pairs: Dict):
+        title = current_div.find('div', class_='col-md-3').text.strip()
+        content = current_div.find('div', class_='col-md-9').text.strip() \
+            .replace('\n', '').replace('\t', '').replace('    ', ' ')
+
+        if title in pairs:
+            if 'date' in pairs[title]:
+                html_items_found[pairs[title]] = content.split(';')
+            elif 'caveat' in pairs[title]:
+                html_items_found[pairs[title]] = content
+                html_items_found['fips_mentioned_certs'] += FIPSCertificate.parse_caveat(content)
+
+            elif 'FIPS Algorithms' in title:
+                html_items_found['fips_algorithms'] += FIPSCertificate.parse_table(
+                    current_div.find('div', class_='col-md-9'))
+
+            elif 'Algorithms' in title:
+                html_items_found['fips_algorithms'] += [{'Certificate': x} for x in
+                                                        FIPSCertificate.parse_algorithms(content)]
+
+            elif 'tested_conf' in pairs[title]:
+                html_items_found[pairs[title]] = [x.text for x in
+                                                  current_div.find('div', class_='col-md-9').find_all('li')]
+            else:
+                html_items_found[pairs[title]] = content
+
+    @staticmethod
+    def parse_vendor(current_div: Tag, html_items_found: Dict, current_file: Path):
+        vendor_string = current_div.find('div', 'panel-body').find('a')
+
+        if not vendor_string:
+            vendor_string = list(current_div.find('div', 'panel-body').children)[0].strip()
+            html_items_found['fips_vendor_www'] = ''
+        else:
+            html_items_found['fips_vendor_www'] = vendor_string.get('href')
+            vendor_string = vendor_string.text.strip()
+
+        html_items_found['fips_vendor'] = vendor_string
+        if html_items_found['fips_vendor'] == '':
+            logging.warning(f"WARNING: NO VENDOR FOUND{current_file}")
+
+    @staticmethod
+    def parse_lab(current_div: Tag, html_items_found: Dict, current_file: Path):
+        html_items_found['fips_lab'] = list(current_div.find('div', 'panel-body').children)[0].strip()
+        html_items_found['fips_nvlap_code'] = \
+            list(current_div.find('div', 'panel-body').children)[2].strip().split('\n')[1].strip()
+
+        if html_items_found['fips_lab'] == '':
+            logging.warning(f"WARNING: NO LAB FOUND{current_file}")
+
+        if html_items_found['fips_nvlap_code'] == '':
+            logging.warning(f"WARNING: NO NVLAP CODE FOUND{current_file}")
+
+    @staticmethod
+    def parse_related_files(current_div: Tag, html_items_found: Dict):
+        links = current_div.find_all('a')
+        ## TODO: break out of circular imports hell
+        html_items_found['fips_security_policy_www'] = __import__(
+            'sec_certs').certificate.FIPSCertificate.FIPS_BASE_URL + links[0].get('href')
+
+        if len(links) == 2:
+            html_items_found['fips_certificate_www'] = __import__(
+                'sec_certs').certificate.FIPSCertificate.FIPS_BASE_URL + links[1].get('href')
+
+    @classmethod
+    def html_from_file(cls, file: Path) -> 'FIPSCertificate':
+        pairs = {
+            'Module Name': 'fips_module_name',
+            'Standard': 'fips_standard',
+            'Status': 'fips_status',
+            'Sunset Date': 'fips_date_sunset',
+            'Validation Dates': 'fips_date_validation',
+            'Overall Level': 'fips_level',
+            'Caveat': 'fips_caveat',
+            'Security Level Exceptions': 'fips_exceptions',
+            'Module Type': 'fips_type',
+            'Embodiment': 'fips_embodiment',
+            'FIPS Algorithms': 'fips_algorithms',
+            'Allowed Algorithms': 'fips_algorithms',
+            'Other Algorithms': 'fips_algorithms',
+            'Tested Configuration(s)': 'fips_tested_conf',
+            'Description': 'fips_description',
+            'Historical Reason': 'fips_historical_reason',
+            'Hardware Versions': 'fips_hw_versions',
+            'Firmware Versions': 'fips_fw_versions'
+        }
+        items_found = FIPSCertificate.initialize_dictionary()
+        items_found['cert_fips_id'] = file.stem
+
+        text = extract_certificates.load_cert_html_file(file)
+        soup = BeautifulSoup(text, 'html.parser')
+        for div in soup.find_all('div', class_='row padrow'):
+            FIPSCertificate.parse_html_main(div, items_found, pairs)
+
+        for div in soup.find_all('div', class_='panel panel-default')[1:]:
+            if div.find('h4', class_='panel-title').text == 'Vendor':
+                FIPSCertificate.parse_vendor(div, items_found, file)
+
+            if div.find('h4', class_='panel-title').text == 'Lab':
+                FIPSCertificate.parse_lab(div, items_found, file)
+
+            if div.find('h4', class_='panel-title').text == 'Related Files':
+                FIPSCertificate.parse_related_files(div, items_found)
+
+        return FIPSCertificate(items_found['cert_fips_id'],
+                               items_found['fips_module_name'],
+                               items_found['fips_standard'],
+                               items_found['fips_status'],
+                               items_found['fips_date_sunset'],
+                               items_found['fips_date_validation'],
+                               items_found['fips_level'],
+                               items_found['fips_caveat'],
+                               items_found['fips_exceptions'],
+                               items_found['fips_type'],
+                               items_found['fips_embodiment'],
+                               items_found['fips_algorithms'],
+                               items_found['fips_tested_conf'],
+                               items_found['fips_description'],
+                               items_found['fips_mentioned_certs'],
+                               items_found['fips_vendor'],
+                               items_found['fips_vendor_www'],
+                               items_found['fips_lab'],
+                               items_found['fips_nvlap_code'],
+                               items_found['fips_historical_reason'],
+                               items_found['fips_security_policy_www'],
+                               items_found['fips_certificate_www'],
+                               items_found['fips_hw_versions'],
+                               items_found['fips_fw_versions'],
+                               False,
+                               None,
+                               [])
 
 
 class CommonCriteriaCert(Certificate):
@@ -136,7 +406,8 @@ class CommonCriteriaCert(Certificate):
         On other values (apart from maintainances, see TODO below) the sanity checks are made.
         """
         if self != other:
-            logging.warning(f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}')
+            logging.warning(
+                f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}')
 
         for att, val in vars(self).items():
             if not val:
@@ -150,7 +421,8 @@ class CommonCriteriaCert(Certificate):
                 pass  # This is expected
             else:
                 if getattr(self, att) != getattr(other, att):
-                    logging.warning(f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}')
+                    logging.warning(
+                        f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}')
         if self.src != other.src:
             self.src = self.src + ' + ' + other.src
 
@@ -170,6 +442,7 @@ class CommonCriteriaCert(Certificate):
         """
         Creates a CC certificate from html row
         """
+
         def get_name(cell: Tag) -> str:
             return list(cell.stripped_strings)[0]
 
@@ -188,14 +461,16 @@ class CommonCriteriaCert(Certificate):
         def get_manufacturer_web(cell: Tag) -> Optional[str]:
             for link in cell.find_all('a'):
                 if link is not None and link.get('title') == 'Vendor\'s web site' and link.get('href') != 'http://':
-                   return link.get('href')
+                    return link.get('href')
             return None
 
         def get_protection_profiles(cell: Tag) -> set:
             protection_profiles = set()
             for link in list(cell.find_all('a')):
                 if link.get('href') is not None and '/ppfiles/' in link.get('href'):
-                    protection_profiles.add(CommonCriteriaCert.ProtectionProfile(str(link.contents[0]), CommonCriteriaCert.cc_url + link.get('href')))
+                    protection_profiles.add(CommonCriteriaCert.ProtectionProfile(str(link.contents[0]),
+                                                                                 CommonCriteriaCert.cc_url + link.get(
+                                                                                     'href')))
             return protection_profiles
 
         def get_date(cell: Tag) -> date:
@@ -242,7 +517,8 @@ class CommonCriteriaCert(Certificate):
                         main_st_link = CommonCriteriaCert.cc_url + l.get('href')
                     else:
                         logging.error('Unknown link in Maintenance part!')
-                maintainance_updates.add(CommonCriteriaCert.MaintainanceReport(main_date, main_title, main_report_link, main_st_link))
+                maintainance_updates.add(
+                    CommonCriteriaCert.MaintainanceReport(main_date, main_title, main_report_link, main_st_link))
             return maintainance_updates
 
         cells = list(row.find_all('td'))
@@ -264,4 +540,5 @@ class CommonCriteriaCert(Certificate):
         maintainance_div = get_maintainance_div(cells[0])
         maintainances = get_maintainance_updates(maintainance_div) if maintainance_div else set()
 
-        return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances)
+        return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link,
+                   st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances)
diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index 76b29390..d8a80fa3 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -34,4 +34,5 @@ TAG_PP_REGISTRATOR_SIMPLIFIED = 'pp_registrator_simplified'
 TAG_PP_SPONSOR = 'pp_sponsor'
 TAG_PP_EDITOR = 'pp_editor'
 TAG_PP_REVIEWER = 'pp_reviewer'
-TAG_KEYWORDS = 'keywords'
\ No newline at end of file
+TAG_KEYWORDS = 'keywords'
+FIPS_NOT_AVAILABLE_CERT_SIZE = 10000
\ No newline at end of file
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 25a59dba..785e3977 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -1,5 +1,12 @@
-from datetime import datetime
-from .certificate import CommonCriteriaCert, Certificate
+import os
+import re
+from datetime import datetime, date
+
+from tabula import read_pdf
+
+from .certificate import CommonCriteriaCert, Certificate, FIPSCertificate
+from .extract_certificates import extract_certificates_keywords
+from .constants import FIPS_NOT_AVAILABLE_CERT_SIZE
 from abc import ABC, abstractmethod
 from . import helpers as helpers
 from pathlib import Path
@@ -8,11 +15,17 @@ import pandas as pd
 from bs4 import BeautifulSoup
 import locale
 import logging
-from typing import Dict
+from typing import Dict, List, Optional, Set, ClassVar
+import json
+from importlib import import_module
+
+from .files import search_files
+from .helpers import find_tables, repair_pdf
 
 
 class Dataset(ABC):
-    def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name', description: str = 'dataset_description'):
+    def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name',
+                 description: str = 'dataset_description'):
         self.root_dir = root_dir
         self.timestamp = datetime.now()
         self.sha256_digest = 'not implemented'
@@ -46,7 +59,8 @@ class Dataset(ABC):
         pass
 
     def to_dict(self):
-        return {'root_dir': self.root_dir, 'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest, 'name': self.name,
+        return {'root_dir': self.root_dir, 'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest,
+                'name': self.name,
                 'description': self.description, 'n_certs': len(self), 'certs': list(self.certs.values())}
 
     @classmethod
@@ -162,13 +176,14 @@ class CCDataset(Dataset):
         """
         Using pandas, this parses a single CSV file.
         """
+
         def get_primary_key_str(row):
             prim_key = row['category'] + row['cert_name'] + row['report_link']
             return prim_key
 
         csv_header = ['category', 'cert_name', 'manufacturer', 'scheme', 'security_level', 'protection_profiles',
-                       'not_valid_before', 'not_valid_after', 'report_link', 'st_link', 'maintainance_date',
-                       'maintainance_title', 'maintainance_report_link', 'maintainance_st_link']
+                      'not_valid_before', 'not_valid_after', 'report_link', 'st_link', 'maintainance_date',
+                      'maintainance_title', 'maintainance_report_link', 'maintainance_st_link']
 
         df = pd.read_csv(file, engine='python', encoding='windows-1250')
         df = df.rename(columns={x: y for (x, y) in zip(list(df.columns), csv_header)})
@@ -176,7 +191,8 @@ class CCDataset(Dataset):
         df['is_maintainance'] = ~df.maintainance_title.isnull()
         df = df.fillna(value='')
 
-        df[['not_valid_before', 'not_valid_after', 'maintainance_date']] = df[['not_valid_before', 'not_valid_after', 'maintainance_date']].apply(pd.to_datetime)
+        df[['not_valid_before', 'not_valid_after', 'maintainance_date']] = df[
+            ['not_valid_before', 'not_valid_after', 'maintainance_date']].apply(pd.to_datetime)
 
         df['dgst'] = df.apply(lambda row: helpers.get_first_16_bytes_sha256(get_primary_key_str(row)), axis=1)
         df_base = df.loc[df.is_maintainance == False].copy()
@@ -190,12 +206,19 @@ class CCDataset(Dataset):
         df_base = df_base.drop_duplicates(subset=['dgst'])
         df_main = df_main.drop_duplicates()
 
-        profiles = {x.dgst: set([CommonCriteriaCert.ProtectionProfile(y, None) for y in helpers.sanitize_protection_profiles(x.protection_profiles)]) for x in df_base.itertuples()}
+        profiles = {x.dgst: set([CommonCriteriaCert.ProtectionProfile(y, None) for y in
+                                 helpers.sanitize_protection_profiles(x.protection_profiles)]) for x in
+                    df_base.itertuples()}
         updates = {x.dgst: set() for x in df_base.itertuples()}
         for x in df_main.itertuples():
-            updates[x.dgst].add(CommonCriteriaCert.MaintainanceReport(x.maintainance_date.date(), x.maintainance_title, x.maintainance_report_link, x.maintainance_st_link))
+            updates[x.dgst].add(CommonCriteriaCert.MaintainanceReport(x.maintainance_date.date(), x.maintainance_title,
+                                                                      x.maintainance_report_link,
+                                                                      x.maintainance_st_link))
 
-        certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level, x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv', None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None)) for x in df_base.itertuples()}
+        certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level,
+                                            x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv',
+                                            None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None)) for x in
+                 df_base.itertuples()}
         return certs
 
     def get_all_certs_from_html(self, get_active, get_archived) -> Dict[str, 'CommonCriteriaCert']:
@@ -218,6 +241,7 @@ class CCDataset(Dataset):
         """
         Prepares a dictionary of certificates from a single html file.
         """
+
         def get_timestamp_from_footer(footer):
             locale.setlocale(locale.LC_ALL, 'en_US')
             footer_text = list(footer.stripped_strings)[0]
@@ -275,3 +299,242 @@ class CCDataset(Dataset):
             certs.update(parse_table(soup, key, val))
 
         return certs
+
+
+class FIPSDataset(Dataset):
+    FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov'
+    FIPS_MODULE_URL: ClassVar[
+        str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
+
+    def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name',
+                 description: str = 'dataset_description'):
+        super().__init__(certs, root_dir, name, description)
+        self.keywords = {}
+        self.new_files = 0
+
+    @property
+    def web_dir(self) -> Path:
+        return self.root_dir / 'web'
+
+    @property
+    def results_dir(self) -> Path:
+        return self.root_dir / 'results'
+
+    @property
+    def policies_dir(self) -> Path:
+        return self.root_dir / 'security_policies'
+
+    @property
+    def fragments_dir(self) -> Path:
+        return self.root_dir / 'fragments'
+
+    def to_dict(self):
+        ## Different - we dont want list
+        return {'root_dir': self.root_dir, 'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest,
+                'name': self.name, 'description': self.description, 'n_certs': len(self),
+                'certs': list(self.certs.values())}
+
+    @classmethod
+    def from_dict(cls, dct: Dict):
+        certs = {x.dgst: x for x in dct['certs']}
+        return cls(certs, dct['root_dir'], dct['name'], dct['description'])
+
+    def find_empty_pdfs(self) -> (List, List):
+        missing = []
+        not_available = []
+        for i in self.certs:
+            if not (self.policies_dir / f'{i}.pdf').exists():
+                missing.append(i)
+            elif os.path.getsize(self.policies_dir / f'{i}.pdf') < FIPS_NOT_AVAILABLE_CERT_SIZE:
+                not_available.append(i)
+        return missing, not_available
+
+    def extract_keywords(self):
+        self.fragments_dir.mkdir(parents=True, exist_ok=True)
+        if self.new_files > 0 or not (self.root_dir / 'fips_full_keywords.json').exists():
+            self.keywords = extract_certificates_keywords(
+                self.policies_dir,
+                self.fragments_dir, 'fips', fips_items=self.certs,
+                should_censure_right_away=True)
+        else:
+            self.keywords = json.loads(open(self.root_dir / 'fips_full_keywords.json').read())
+
+    def dump_to_json(self):
+        with open(self.root_dir / 'fips_full_dataset.json', 'w') as handle:
+            json.dump(self, handle, cls=import_module('sec_certs.serialization').CustomJSONEncoder, indent=4)
+
+    def dump_keywords(self):
+        with open(self.root_dir / "fips_full_keywords.json", 'w') as f:
+            f.write(json.dumps(self.keywords, indent=4, sort_keys=True))
+
+    # TODO figure out whether the name of this method shuold not be "get_certs", because we don't download every time
+
+    def get_certs_from_web(self):
+        def get_certificates_from_html(html_file: Path) -> None:
+            logging.info(f'Getting certificate ids from {html_file}')
+            html = BeautifulSoup(open(html_file).read(), 'html.parser')
+
+            table = [x for x in html.find(id='searchResultsTable').tbody.contents if x != '\n']
+            for entry in table:
+                self.certs[entry.find('a').text] = {}
+
+        logging.info("Downloading required html files")
+
+        self.web_dir.mkdir(parents=True, exist_ok=True)
+        self.policies_dir.mkdir(exist_ok=True)
+
+        # Download files containing all available module certs (always)
+        html_files = ['fips_modules_active.html', 'fips_modules_historical.html', 'fips_modules_revoked.html']
+        helpers.download_file(
+            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0",
+            self.web_dir / "fips_modules_active.html")
+        helpers.download_file(
+            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Historical&ValidationYear=0",
+            self.web_dir / "fips_modules_historical.html")
+        helpers.download_file(
+            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Revoked&ValidationYear=0",
+            self.web_dir / "fips_modules_revoked.html")
+
+        # Parse those files and get list of currently processable files (always)
+        for f in html_files:
+            get_certificates_from_html(self.web_dir / f)
+
+        logging.info('Downloading certficate html and security policies')
+        html_items = [
+            (f"https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{cert_id}",
+             self.web_dir / f"{cert_id}.html") for cert_id in list(self.certs.keys()) if
+            not (self.web_dir / f'{cert_id}.html').exists()]
+        sp_items = [(
+            f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf",
+            self.policies_dir / f"{cert_id}.pdf") for cert_id in list(self.certs.keys()) if
+            not (self.policies_dir / f'{cert_id}.pdf').exists()]
+
+        _, self.new_files = helpers.download_parallel(html_items + sp_items, 8), len(html_items) + len(sp_items)
+
+        logging.info(f"{self.new_files} needed to be downloaded")
+
+        if self.new_files > 0 or not (self.root_dir / 'fips_full_dataset.json').exists():
+            # if False:
+            for cert in self.certs:
+                self.certs[cert] = FIPSCertificate.html_from_file(self.web_dir / f'{cert}.html')
+        else:
+            logging.info("Certs loaded from previous scanning")
+            dataset = json.loads(open(self.root_dir / 'fips_full_dataset.json').read(),
+                                 cls=import_module('sec_certs.serialization').CustomJSONDecoder)
+            self.certs = dataset.certs
+
+    def extract_certs_from_tables(self) -> List[Path]:
+        """
+        Function that extracts algorithm IDs from tables in security policies files.
+        :return: list of files that couldn't have been decoded
+        """
+
+        list_of_files = search_files(self.policies_dir)
+        not_decoded = []
+        for cert_file in list_of_files:
+            cert_file = Path(cert_file)
+
+            if '.txt' not in cert_file.suffixes:
+                continue
+
+            stem_name = Path(cert_file.stem).stem
+
+            if self.certs[stem_name].tables_done:
+                continue
+
+            with open(cert_file, 'r') as f:
+                tables = find_tables(f.read(), cert_file)
+
+            # If we find any tables with page numbers, we process them
+            if tables:
+                lst = []
+                try:
+                    data = read_pdf(cert_file.with_suffix(''), pages=tables, silent=True)
+                except Exception:
+                    try:
+                        repair_pdf(cert_file.with_suffix(''))
+                        data = read_pdf(cert_file.with_suffix(''), pages=tables, silent=True)
+
+                    except Exception:
+                        not_decoded.append(cert_file)
+                        continue
+
+                # find columns with cert numbers
+                for df in data:
+                    for col in range(len(df.columns)):
+                        if 'cert' in df.columns[col].lower() or 'algo' in df.columns[col].lower():
+                            lst += FIPSCertificate.parse_algorithms(df.iloc[:, col].to_string(index=False), True)
+
+                    # Parse again if someone picks not so descriptive column names
+                    lst += FIPSCertificate.parse_algorithms(df.to_string(index=False))
+
+                if lst:
+                    self.certs[stem_name].algorithms += lst
+
+            self.certs[stem_name].tables_done = True
+        return not_decoded
+
+    def remove_algorithms_from_extracted_data(self):
+        """
+        Function that removes all found certificate IDs that are matching any IDs labeled as algorithm IDs
+        """
+        for file_name in self.keywords:
+            self.keywords[file_name]['file_status'] = True
+            self.certs[file_name].file_status = True
+            if self.certs[file_name].mentioned_certs:
+                for item in self.certs[file_name].mentioned_certs:
+                    self.keywords[file_name]['rules_cert_id'].update(item)
+
+            for rule in self.keywords[file_name]['rules_cert_id']:
+                to_pop = set()
+                rr = re.compile(rule)
+                for cert in self.keywords[file_name]['rules_cert_id'][rule]:
+                    for alg in self.keywords[file_name]['rules_fips_algorithms']:
+                        for found in self.keywords[file_name]['rules_fips_algorithms'][alg]:
+                            if rr.search(found) and rr.search(cert) and rr.search(found).group('id') == rr.search(
+                                    cert).group('id'):
+                                to_pop.add(cert)
+                for r in to_pop:
+                    self.keywords[file_name]['rules_cert_id'][rule].pop(r, None)
+
+                self.keywords[file_name]['rules_cert_id'][rule].pop(
+                    self.certs[file_name].cert_id, None)
+
+    def validate_results(self):
+        """
+        Function that validates results and finds the final connection output
+        """
+        broken_files = set()
+        for file_name in self.keywords:
+            for rule in self.keywords[file_name]['rules_cert_id']:
+                for cert in self.keywords[file_name]['rules_cert_id'][rule]:
+                    cert_id = ''.join(filter(str.isdigit, cert))
+
+                    if cert_id == '' or cert_id not in self.certs:
+                        # TEST
+                        # if cert_id == '' or int(cert_id) > 3730:
+                        broken_files.add(file_name)
+                        self.keywords[file_name]['file_status'] = False
+                        self.certs[file_name].file_status = False
+                        break
+        if broken_files:
+            logging.warning("CERTIFICATE FILES WITH WRONG CERTIFICATES PARSED")
+            logging.warning(broken_files)
+            logging.warning("... skipping these...")
+            logging.warning(f"Total non-analyzable files:{len(broken_files)}")
+
+        for file_name in self.keywords:
+            self.certs[file_name].connections = []
+            if not self.keywords[file_name]['file_status']:
+                continue
+            if self.keywords[file_name]['rules_cert_id'] == {}:
+                continue
+            for rule in self.keywords[file_name]['rules_cert_id']:
+                for cert in self.keywords[file_name]['rules_cert_id'][rule]:
+                    cert_id = ''.join(filter(str.isdigit, cert))
+                    if cert_id not in self.certs[file_name].connections:
+                        self.certs[file_name].connections.append(cert_id)
+
+    def finalize_results(self):
+        self.remove_algorithms_from_extracted_data()
+        self.validate_results()
diff --git a/sec_certs/download.py b/sec_certs/download.py
index dccbef40..16ac91de 100644
--- a/sec_certs/download.py
+++ b/sec_certs/download.py
@@ -2,7 +2,7 @@ import os
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
 from tqdm import tqdm
-from typing import Sequence, Tuple
+from typing import Sequence, Tuple, List
 
 import requests
 
@@ -22,6 +22,7 @@ def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Se
     def download(url_output):
         url, output = url_output
         return url, download_file(url, output)
+
     pool = ThreadPool(num_threads)
     responses = []
     with tqdm(total=len(items)) as progress:
@@ -37,20 +38,20 @@ def download_cc_web(web_dir: Path, num_threads: int) -> Sequence[Tuple[str, int]
     items = [
         ("https://www.commoncriteriaportal.org/products/", web_dir / "cc_products_active.html"),
         ("https://www.commoncriteriaportal.org/products/index.cfm?archived=1",
-            web_dir / "cc_products_archived.html"),
+         web_dir / "cc_products_archived.html"),
         ("https://www.commoncriteriaportal.org/labs/", web_dir / "cc_labs.html"),
         ("https://www.commoncriteriaportal.org/products/certified_products.csv",
-            web_dir / "cc_products_active.csv"),
+         web_dir / "cc_products_active.csv"),
         ("https://www.commoncriteriaportal.org/products/certified_products-archived.csv",
-            web_dir / "cc_products_archived.csv"),
+         web_dir / "cc_products_archived.csv"),
         ("https://www.commoncriteriaportal.org/pps/", web_dir / "cc_pp_active.html"),
         ("https://www.commoncriteriaportal.org/pps/collaborativePP.cfm?cpp=1",
-            web_dir / "cc_pp_collaborative.html"),
+         web_dir / "cc_pp_collaborative.html"),
         ("https://www.commoncriteriaportal.org/pps/index.cfm?archived=1",
-            web_dir / "cc_pp_archived.html"),
+         web_dir / "cc_pp_archived.html"),
         ("https://www.commoncriteriaportal.org/pps/pps.csv", web_dir / "cc_pp_active.csv"),
         ("https://www.commoncriteriaportal.org/pps/pps-archived.csv",
-            web_dir / "cc_pp_archived.csv")]
+         web_dir / "cc_pp_archived.csv")]
     return download_parallel(items, num_threads)
 
 
@@ -102,15 +103,28 @@ def download_cc_failed(walk_dir: Path, num_threads: int) -> Sequence[Tuple[str,
 
 
 def download_fips_web(web_dir: Path):
-    download_file("https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search/all",
-                  web_dir / "fips_modules_validated.html")
+    download_file(
+        "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0",
+        web_dir / "fips_modules_active.html")
+    download_file(
+        "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Historical&ValidationYear=0",
+        web_dir / "fips_modules_historical.html")
+    download_file(
+        "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Revoked&ValidationYear=0",
+        web_dir / "fips_modules_revoked.html")
+
 
+def download_fips(web_dir: Path, policies_dir: Path, num_threads: int, ids: List[str]) \
+        -> Tuple[Sequence[Tuple[str, int]], int]:
+    web_dir.mkdir(exist_ok=True)
+    policies_dir.mkdir(exist_ok=True)
 
-def download_fips(web_dir: Path, policies_dir: Path, num_threads: int) -> Sequence[Tuple[str, int]]:
     html_items = [
         (f"https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{cert_id}",
-         web_dir / f"{cert_id}.html") for cert_id in range(1, 4001)]
+         web_dir / f"{cert_id}.html") for cert_id in ids if not (web_dir / f'{cert_id}.html').exists()]
     sp_items = [
-        (f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf",
-         policies_dir / f"{cert_id}.pdf") for cert_id in range(1, 4001)]
-    return download_parallel(html_items + sp_items, num_threads)
+        (
+            f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf",
+            policies_dir / f"{cert_id}.pdf") for cert_id in ids if not (policies_dir / f'{cert_id}.pdf').exists()
+    ]
+    return download_parallel(html_items + sp_items, num_threads), len(html_items) + len(sp_items)
diff --git a/sec_certs/extract_certificates.py b/sec_certs/extract_certificates.py
index d03b18d6..e4192855 100644
--- a/sec_certs/extract_certificates.py
+++ b/sec_certs/extract_certificates.py
@@ -165,7 +165,7 @@ def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=
                 is_algorithm = False
                 if fips_items and match != '':
                     certs = [x['Certificate']
-                             for x in fips_items[file_name]['fips_algorithms']]
+                             for x in fips_items[file_name].algorithms]
 
                     match_cert_id = ''.join(filter(str.isdigit, match))
                     #                    if file_name == '/home/stan/sec-certs-master/files/fips/security_policies/3676.html.txt':
diff --git a/sec_certs/files.py b/sec_certs/files.py
index dcd3262f..194ece6b 100644
--- a/sec_certs/files.py
+++ b/sec_certs/files.py
@@ -32,5 +32,3 @@ def load_json_files(files_list):
             loaded_jsons.append(loaded_items)
             print('{} loaded, total items = {}'.format(file_name, len(loaded_items)))
     return tuple(loaded_jsons)
-
-
diff --git a/sec_certs/fips_certificates.py b/sec_certs/fips_certificates.py
index a6a73bf0..22b055a8 100755
--- a/sec_certs/fips_certificates.py
+++ b/sec_certs/fips_certificates.py
@@ -19,18 +19,6 @@ from .files import load_json_files, FILE_ERRORS_STRATEGY, search_files
 FIPS_BASE_URL = 'https://csrc.nist.gov'
 FIPS_MODULE_URL = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
 
-
-def find_empty_pdfs(base_dir: Path) -> (List, List):
-    missing = []
-    not_available = []
-    for i in range(1, 3725):
-        if not (base_dir / f'{i}.pdf').exists():
-            missing.append(i)
-        elif os.path.getsize(base_dir / f'{i}.pdf') < 10000:
-            not_available.append(i)
-    return missing, not_available
-
-
 def extract_filename(file: str) -> str:
     """
     Extracts filename from path
@@ -40,150 +28,8 @@ def extract_filename(file: str) -> str:
     return os.path.splitext(os.path.basename(file))[0]
 
 
-def parse_table(element: BeautifulSoup) -> List[Dict]:
-    """
-    Parses content of <table> tags in FIPS .html CMVP page
-    :param element: text in <table> tags
-    :return: list of all found algorithm IDs
-    """
-    found_items = []
-    trs = element.find_all('tr')
-    for tr in trs:
-        tds = tr.find_all('td')
-        found_items.append({'Name': tds[0].text, 'Certificate': parse_algorithms(tds[1].text)})
-
-    return found_items
-
-
-def parse_algorithms(text: str, in_pdf: bool = False) -> List:
-    """
-    Parses table of FIPS (non) allowed algorithms
-    :param text: Contents of the table
-    :param in_pdf: Specifies whether the table was found in a PDF security policies file
-    :return: list of all found algorithm IDs
-    """
-    set_items = set()
-    for m in re.finditer(rf"(?:#{'?' if in_pdf else 'C?'}\s?|Cert\.?[^. ]*?\s?)(?:[Cc]\s)?(?P<id>\d+)", text):
-        set_items.add(m.group())
-
-    return list(set_items)
-
-
-def parse_caveat(text: str) -> List:
-    """
-    Parses content of "Caveat" of FIPS CMVP .html file
-    :param text: text of "Caveat"
-    :return: list of all found algorithm IDs
-    """
-    items_found = []
-    r_key = r"(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)(?P<id>\d+)"
-    for m in re.finditer(r_key, text):
-        if r_key in items_found and m.group() in items_found[0]:
-            items_found[0][m.group()]['count'] += 1
-        else:
-            items_found.append({r"(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)(?P<id>\d+?})": {m.group(): {'count': 1}}})
-
-    return items_found
-
-
-def initialize_entry(input_dictionary: Dict):
-    """
-    Initialize input dictionary with elements that should be always processed
-    :param input_dictionary: empty dictionary used as "all_items"
-    """
-    input_dictionary['fips_exceptions'] = []
-    input_dictionary['fips_tested_conf'] = []
-    input_dictionary['fips_mentioned_certs'] = []
-
-    input_dictionary['fips_algorithms'] = []
-    input_dictionary['fips_caveat'] = []
-    input_dictionary['tables_done'] = False
-    input_dictionary['fips_module_name'] = ''
-
-
-def fips_search_html(base_dir: Path, output_file: str, dump_to_file: bool = False) -> Dict:
-    all_found_items = {}
-    pairs = {
-        'Module Name': 'fips_module_name',
-        'Standard': 'fips_standard',
-        'Status': 'fips_status',
-        'Sunset Date': 'fips_date_sunset',
-        'Validation Dates': 'fips_date_validation',
-        'Overall Level': 'fips_level',
-        'Caveat': 'fips_caveat',
-        'Security Level Exceptions': 'fips_exceptions',
-        'Module Type': 'fips_type',
-        'Embodiment': 'fips_embodiment',
-        'FIPS Algorithms': 'fips_algorithms',
-        'Allowed Algorithms': 'fips_algorithms',
-        'Other Algorithms': 'fips_algorithms',
-        'Tested Configuration(s)': 'fips_tested_conf',
-        'Description': 'fips_description'
-    }
-
-    for file in search_files(base_dir):
-        current_items_found = {'cert_fips_id': extract_filename(file)}
-        all_found_items[extract_filename(file)] = current_items_found
-        initialize_entry(current_items_found)
-        text = extract_certificates.load_cert_html_file(file)
-        soup = BeautifulSoup(text, 'html.parser')
-        for div in soup.find_all('div', class_='row padrow'):
-            title = div.find('div', class_='col-md-3').text.strip()
-            content = div.find('div', class_='col-md-9').text.strip() \
-                .replace('\n', '').replace('\t', '').replace('    ', ' ')
-
-            if title in pairs:
-                if 'date' in pairs[title]:
-                    current_items_found[pairs[title]] = content.split(';')
-                elif 'caveat' in pairs[title]:
-                    current_items_found[pairs[title]] = content
-                    current_items_found['fips_mentioned_certs'] += parse_caveat(content)
-
-                elif 'FIPS Algorithms' in title:
-                    current_items_found['fips_algorithms'] += parse_table(div.find('div', class_='col-md-9'))
-
-                elif 'Algorithms' in title:
-                    current_items_found['fips_algorithms'] += [{'Certificate': x} for x in parse_algorithms(content)]
-
-                elif 'tested_conf' in pairs[title]:
-                    current_items_found[pairs[title]] = [x.text for x in
-                                                         div.find('div', class_='col-md-9').find_all('li')]
-                else:
-                    current_items_found[pairs[title]] = content
-
-        for div in soup.find_all('div', class_='panel panel-default')[1:]:
-            if div.find('h4', class_='panel-title').text == 'Vendor':
-                vendor_string = div.find('div', 'panel-body').find('a')
-                if not vendor_string:
-                    vendor_string = list(div.find('div', 'panel-body').children)[0].strip()
-                    current_items_found['fips_vendor_www'] = ''
-                else:
-                    current_items_found['fips_vendor_www'] = vendor_string.get('href')
-                    vendor_string = vendor_string.text.strip()
-                current_items_found['fips_vendor'] = vendor_string
-                if current_items_found['fips_vendor'] == '':
-                    print("WARNING: NO VENDOR FOUND", file)
-
-            if div.find('h4', class_='panel-title').text == 'Lab':
-                current_items_found['fips_lab'] = list(div.find('div', 'panel-body').children)[0].strip()
-                current_items_found['fips_nvlap_code'] = \
-                    list(div.find('div', 'panel-body').children)[2].strip().split('\n')[1].strip()
-                if current_items_found['fips_lab'] == '':
-                    print("WARNING: NO LAB FOUND", file)
-                if current_items_found['fips_nvlap_code'] == '':
-                    print("WARNING: NO NVLAP CODE FOUND", file)
-
-            if div.find('h4', class_='panel-title').text == 'Related Files':
-                links = div.find_all('a')
-                current_items_found['fips_security_policy_www'] = FIPS_BASE_URL + links[0].get('href')
-                if len(links) == 2:
-                    current_items_found['fips_certificate_www'] = FIPS_BASE_URL + links[1].get('href')
-
-    if dump_to_file:
-        with open(output_file, 'w', errors=FILE_ERRORS_STRATEGY) as write_file:
-            json.dump(all_found_items, write_file, indent=4, sort_keys=True)
-
-    return all_found_items
+def initialize_entry(current_items_found):
+    pass
 
 
 def get_dot_graph(found_items: Dict, output_file_name: str):
@@ -251,32 +97,7 @@ def get_dot_graph(found_items: Dict, output_file_name: str):
 
 
 def remove_algorithms_from_extracted_data(items, html):
-    """
-    Function that removes all found certificate IDs that are matching any IDs labeled as algorithm IDs
-    :param items: All keyword items found in pdf files
-    :param html: All items extracted from html files
-    """
-    for file_name in items:
-        items[file_name]['file_status'] = True
-        html[file_name]['file_status'] = True
-        if html[file_name]['fips_mentioned_certs']:
-            for item in html[file_name]['fips_mentioned_certs']:
-                items[file_name]['rules_cert_id'].update(item)
-
-        for rule in items[file_name]['rules_cert_id']:
-            to_pop = set()
-            rr = re.compile(rule)
-            for cert in items[file_name]['rules_cert_id'][rule]:
-                for alg in items[file_name]['rules_fips_algorithms']:
-                    for found in items[file_name]['rules_fips_algorithms'][alg]:
-                        if rr.search(found) and rr.search(cert) and rr.search(found).group('id') == rr.search(
-                                cert).group('id'):
-                            to_pop.add(cert)
-            for r in to_pop:
-                items[file_name]['rules_cert_id'][rule].pop(r, None)
-
-            items[file_name]['rules_cert_id'][rule].pop(
-                html[file_name]['cert_fips_id'], None)
+    pass
 
 
 def validate_results(items: Dict, html: Dict):
@@ -318,16 +139,7 @@ def validate_results(items: Dict, html: Dict):
 
 
 def parse_list_of_tables(txt: str) -> Set[str]:
-    """
-    Parses list of tables from function find_tables(), finds ones that mention algorithms
-    :param txt: chunk of text
-    :return: set of all pages mentioning algorithm table
-    """
-    rr = re.compile(r"^.+?(?:[Ff]unction|[Aa]lgorithm).+?(?P<page_num>\d+)$", re.MULTILINE)
-    pages = set()
-    for m in rr.finditer(txt):
-        pages.add(m.group('page_num'))
-    return pages
+    pass
 
 
 def extract_page_number(txt: str) -> Optional[str]:
@@ -354,16 +166,7 @@ def extract_page_number(txt: str) -> Optional[str]:
 
 
 def find_tables_iterative(file_text: str) -> List[int]:
-    current_page = 1
-    pages = set()
-    for line in file_text.split('\n'):
-        if '\f' in line:
-            current_page += 1
-        if line.startswith('Table ') or line.startswith('Exhibit'):
-            pages.add(current_page)
-    if not pages:
-        print('~' * 20, 'No pages found', '~' * 20)
-    return list(pages)
+    pass
 
 
 def find_footers(txt: str, num_pages: int) -> Optional[List]:
@@ -392,87 +195,15 @@ def find_footers(txt: str, num_pages: int) -> Optional[List]:
 
 
 def find_tables(txt: str, file_name: Path) -> Optional[List]:
-    """
-    Function that tries to pages in security policy pdf files, where it's possible to find a table containing
-    algorithms
-    :param txt: file in .txt format (output of pdftotext)
-    :param file_name: name of the file
-    :return:    list of pages possibly containing a table
-                None if these cannot be found
-    """
-    # Look for "List of Tables", where we can find exactly tables with page num
-    tables_regex = re.compile(r"^(?:(?:[Tt]able\s|[Ll]ist\s)(?:[Oo]f\s))[Tt]ables[\s\S]+?\f", re.MULTILINE)
-    table = tables_regex.search(txt)
-    if table:
-        rb = parse_list_of_tables(table.group())
-        if rb:
-            return list(rb)
-        return None
+    pass
 
-    # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer
-    print("~" * 20, file_name, '~' * 20)
-    rb = find_tables_iterative(txt)
-    return rb if rb else None
 
-
-def repair_pdf(file: Path):
-    """
-    Some pdfs can't be opened by PyPDF2 - opening them with pikepdf and then saving them fixes this issue.
-    By opening this file in a pdf reader, we can already extract number of pages
-    :param file: file name
-    :return: number of pages in pdf file
-    """
-    pdf = pikepdf.Pdf.open(file, allow_overwriting_input=True)
-    pdf.save(file)
+def parse_algorithms(a, b=False):
+    pass
 
 
 def extract_certs_from_tables(list_of_files: List, html_items: Dict) -> List[Path]:
-    """
-    Function that extracts algorithm IDs from tables in security policies files.
-    :param list_of_files: iterable containing all files to parse
-    :param html_items: dictionary created by main() containing data extracted from html pages
-    :return: list of files that couldn't have been decoded
-    """
-    not_decoded = []
-    for cert_file in list_of_files:
-        if '.txt' not in cert_file:
-            continue
-
-        if html_items[extract_filename(cert_file[:-8])]['tables_done']:
-            continue
-
-        with open(cert_file, 'r') as f:
-            tables = find_tables(f.read(), cert_file)
-
-        # If we find any tables with page numbers, we process them
-        if tables:
-            lst = []
-            print("~~~~~~~~~~~~~~~", cert_file, "~~~~~~~~~~~~~~~~~~~~~~~")
-            try:
-                data = read_pdf(cert_file[:-4], pages=tables, silent=True)
-            except Exception:
-                try:
-                    repair_pdf(cert_file[:-4])
-                    data = read_pdf(cert_file[:-4], pages=tables, silent=True)
-
-                except Exception:
-                    not_decoded.append(cert_file)
-                    continue
-
-            # find columns with cert numbers
-            for df in data:
-                for col in range(len(df.columns)):
-                    if 'cert' in df.columns[col].lower() or 'algo' in df.columns[col].lower():
-                        lst += parse_algorithms(df.iloc[:, col].to_string(index=False), True)
-
-                # Parse again if someone picks not so descriptive column names
-                lst += parse_algorithms(df.to_string(index=False))
-
-            if lst:
-                html_items[extract_filename(cert_file[:-8])]['fips_algorithms'] += lst
-
-        html_items[extract_filename(cert_file[:-8])]['tables_done'] = True
-    return not_decoded
+    pass
 
 
 @click.command()
@@ -500,9 +231,8 @@ def main(directory, do_download_meta: bool, do_download_certs: bool, threads: in
     if do_download_certs:
         download_fips(web_dir, policies_dir, threads)
 
-    missing, not_available = find_empty_pdfs(policies_dir)
-    print(f"Missing security policies: Total {len(missing)}")
-    print(f"Not available security policies: Total {len(not_available)}")
+    print(f"Missing security policies: Total {len([])}")
+    print(f"Not available security policies: Total {len([])}")
     files_to_load = [
         results_dir / 'fips_data_keywords_all.json',
         results_dir / 'fips_html_all.json'
@@ -510,11 +240,9 @@ def main(directory, do_download_meta: bool, do_download_certs: bool, threads: in
 
     for file in files_to_load:
         if not os.path.isfile(file):
-            fips_items = fips_search_html(web_dir,
-                                          results_dir / 'fips_html_all.json', True)
             items = extract_certificates.extract_certificates_keywords(
                 policies_dir,
-                fragments_dir, 'fips', fips_items=fips_items,
+                fragments_dir, 'fips', fips_items=None,
                 should_censure_right_away=True)
             with open(results_dir / 'fips_data_keywords_all.json', 'w') as f:
                 json.dump(items, f, indent=4, sort_keys=True)
diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py
index da5e73d7..50518911 100644
--- a/sec_certs/helpers.py
+++ b/sec_certs/helpers.py
@@ -1,4 +1,8 @@
-from typing import Sequence, Tuple
+import re
+from typing import Sequence, Tuple, Optional, Set, List, Dict
+
+import logging
+import pikepdf
 import requests
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
@@ -9,6 +13,8 @@ from typing import Union
 from datetime import date
 import numpy as np
 import pandas as pd
+from bs4 import Tag, NavigableString
+
 
 
 def download_file(url: str, output: Path) -> int:
@@ -62,7 +68,6 @@ def sanitize_string(record: str) -> Union[str, None]:
         return ' '.join(string.split())
 
 
-
 def sanitize_security_levels(record: Union[str, set]) -> set:
     if isinstance(record, str):
         record = set(record.split(','))
@@ -79,4 +84,72 @@ def sanitize_security_levels(record: Union[str, set]) -> set:
 def sanitize_protection_profiles(record: str) -> list:
     if not record:
         return []
-    return record.split(',')
\ No newline at end of file
+    return record.split(',')
+
+
+# TODO: realize whether this stays or goes somewhere else
+def parse_list_of_tables(txt: str) -> Set[str]:
+    """
+    Parses list of tables from function find_tables(), finds ones that mention algorithms
+    :param txt: chunk of text
+    :return: set of all pages mentioning algorithm table
+    """
+    rr = re.compile(r"^.+?(?:[Ff]unction|[Aa]lgorithm).+?(?P<page_num>\d+)$", re.MULTILINE)
+    pages = set()
+    for m in rr.finditer(txt):
+        pages.add(m.group('page_num'))
+    return pages
+
+
+def find_tables_iterative(file_text: str) -> List[int]:
+    current_page = 1
+    pages = set()
+    for line in file_text.split('\n'):
+        if '\f' in line:
+            current_page += 1
+        if line.startswith('Table ') or line.startswith('Exhibit'):
+            pages.add(current_page)
+    if not pages:
+        logging.warning('No pages found')
+    return list(pages)
+
+
+def find_tables(txt: str, file_name: Path) -> Optional[List]:
+    """
+    Function that tries to pages in security policy pdf files, where it's possible to find a table containing
+    algorithms
+    :param txt: file in .txt format (output of pdftotext)
+    :param file_name: name of the file
+    :return:    list of pages possibly containing a table
+                None if these cannot be found
+    """
+    # Look for "List of Tables", where we can find exactly tables with page num
+    tables_regex = re.compile(r"^(?:(?:[Tt]able\s|[Ll]ist\s)(?:[Oo]f\s))[Tt]ables[\s\S]+?\f", re.MULTILINE)
+    table = tables_regex.search(txt)
+    if table:
+        rb = parse_list_of_tables(table.group())
+        if rb:
+            return list(rb)
+        return None
+
+    # Otherwise look for "Table" in text and \f representing footer, then extract page number from footer
+    logging.info(f'parsing tables in {file_name}')
+    rb = find_tables_iterative(txt)
+    return rb if rb else None
+
+
+def repair_pdf(file: Path):
+    """
+    Some pdfs can't be opened by PyPDF2 - opening them with pikepdf and then saving them fixes this issue.
+    By opening this file in a pdf reader, we can already extract number of pages
+    :param file: file name
+    :return: number of pages in pdf file
+    """
+    pdf = pikepdf.Pdf.open(file, allow_overwriting_input=True)
+    pdf.save(file)
+
+
+
+
+
+
diff --git a/sec_certs/serialization.py b/sec_certs/serialization.py
index 2fb77110..8283ac35 100644
--- a/sec_certs/serialization.py
+++ b/sec_certs/serialization.py
@@ -1,10 +1,12 @@
 import json
 from datetime import date
 from pathlib import Path
-from .dataset import CCDataset
-from .certificate import CommonCriteriaCert
+from .dataset import CCDataset, FIPSDataset
+from .certificate import CommonCriteriaCert, FIPSCertificate
 
-serializable_complex_types = (CCDataset, CommonCriteriaCert, CommonCriteriaCert.MaintainanceReport, CommonCriteriaCert.ProtectionProfile)
+serializable_complex_types = (
+CCDataset, FIPSDataset, CommonCriteriaCert, CommonCriteriaCert.MaintainanceReport, CommonCriteriaCert.ProtectionProfile,
+FIPSCertificate)
 serializable_complex_types_dict = {x.__name__: x for x in serializable_complex_types}
 
 
@@ -29,3 +31,5 @@ class CustomJSONDecoder(json.JSONDecoder):
         if '_type' in obj and obj['_type'] in serializable_complex_types_dict.keys():
             complex_type = obj.pop('_type')
             return serializable_complex_types_dict[complex_type].from_dict(obj)
+
+        return obj
diff --git a/test/test_download.py b/test/test_download.py
index 59fef407..8c83c432 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -24,7 +24,9 @@ class BasicTests(TestCase):
         with TemporaryDirectory() as tmp_dir:
             tmp_path = Path(tmp_dir)
             download_fips_web(tmp_path)
-            fips_files = {"fips_modules_validated.html"}
+            fips_files = {'fips_modules_active.html',
+                          'fips_modules_historical.html',
+                          'fips_modules_revoked.html'}
             actual = {path.name for path in tmp_path.iterdir()}
             self.assertEqual(fips_files, actual)
 
@@ -32,7 +34,7 @@ class BasicTests(TestCase):
         with open(self.test_data_dir / "certs.csv") as f:
             reader = csv.DictReader(f)
             certs = [(row["cert"], row["st"]) for row in reader]
-        cert_list = [( "/epfiles/" + cert, cert, "/epfiles/" + st, st) for cert, st in certs]
+        cert_list = [("/epfiles/" + cert, cert, "/epfiles/" + st, st) for cert, st in certs]
         with TemporaryDirectory() as tmp_dir:
             tmp_path = Path(tmp_dir)
             (tmp_path / "certs").mkdir()