Alpha version of OOP certificates.

Currently missing features:
- Tests
- Sample script
- Exceptions
- Robbust logging
- Acive/Archived certificate distinguisher
- Some methods are not implemented

6 files changed, 522 insertions, 138 deletions

diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index a4ff8288..2a39bd43 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -1,53 +1,247 @@
 from typing import Type
+from datetime import datetime, date
+from dataclasses import dataclass
 import json
-
+import logging
+from . import helpers
+from typing import Union
 from abc import ABC, abstractmethod
-from . import constants as constants
-
-
-class CertificateJSONEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, Certificate):
-            return obj.to_dict()
-        return super().default(obj)
+from bs4 import Tag
+from typing import Union
 
 
 class Certificate(ABC):
     def __init__(self):
-        self.sha256 = None
+        pass
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return str(self.to_dict())
 
-    def __str__(self):
+    def __str__(self) -> str:
         return 'Not implemented'
 
     @abstractmethod
-    def to_dict(self):
+    def to_dict(self) -> dict:
         raise NotImplementedError('Not meant to be implemented')
 
     def __eq__(self, other: 'Certificate') -> bool:
-        return self.sha256 == other.sha256
+        return self.dgst == other.dgst
 
     @classmethod
     @abstractmethod
-    def from_dict(cls, dct):
+    def from_dict(cls, dct: dict) -> 'Certificate':
         raise NotImplementedError('Mot meant to be implemented')
 
 
-class CommonCriteriaCert(Certificate):
-    def to_dict(self):
+class FIPSCertificate(Certificate):
+    def to_dict(self) -> dict:
         pass
 
     @classmethod
-    def from_dict(cls, dct):
-        return CommonCriteriaCert()
+    def from_dict(cls, dct: dict) -> 'FIPSCertificate':
+        return FIPSCertificate()
 
 
-class FIPSCertificate(Certificate):
-    def to_dict(self):
+class CommonCriteriaCert(Certificate):
+    cc_url = 'http://www.commoncriteriaportal.org'
+
+    @dataclass(eq=True, frozen=True)
+    class MaintainanceReport:
+        """
+        Object for holding maintainance reports.
+        """
+        maintainance_date: date
+        maintainance_title: str
+        maintainance_report_link: str
+        maintainance_st_link: str
+
+        def __post_init__(self):
+            super().__setattr__('maintainance_report_link', helpers.sanitize_link(self.maintainance_report_link))
+            super().__setattr__('maintainance_st_link', helpers.sanitize_link(self.maintainance_st_link))
+            super().__setattr__('maintainance_title', helpers.sanitize_string(self.maintainance_title))
+            super().__setattr__('maintainance_date', helpers.sanitize_date(self.maintainance_date))
+
+        def to_dict(self):
+            return self.__dict__
+
+    @dataclass(eq=True, frozen=True)
+    class ProtectionProfile:
+        """
+        Object for holding protection profiles.
+        """
+        name: str
+        link: Union[str, None]
+
+        def __post_init__(self):
+            super().__setattr__('name', helpers.sanitize_string(self.name))
+            super().__setattr__('link', helpers.sanitize_link(self.link))
+
+        def to_dict(self):
+            return self.__dict__
+
+    def __init__(self, category: str, name: str, manufacturer: str, scheme: str,
+                 security_level: Union[str, set], not_valid_before: date,
+                 not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Union[str, None],
+                 manufacturer_web: Union[str, None],
+                 protection_profiles: set,
+                 maintainance_updates: set):
+        super().__init__()
+
+        self.category = category
+        self.name = helpers.sanitize_string(name)
+        self.manufacturer = helpers.sanitize_string(manufacturer)
+        self.scheme = scheme
+        self.security_level = helpers.sanitize_security_levels(security_level)
+        self.not_valid_before = helpers.sanitize_date(not_valid_before)
+        self.not_valid_after = helpers.sanitize_date(not_valid_after)
+        self.report_link = helpers.sanitize_link(report_link)
+        self.st_link = helpers.sanitize_link(st_link)
+        self.src = src
+        self.cert_link = helpers.sanitize_link(cert_link)
+        self.manufacturer_web = helpers.sanitize_link(manufacturer_web)
+        self.protection_profiles = protection_profiles
+        self.maintainances = maintainance_updates
+
+    @property
+    def dgst(self) -> str:
+        """
+        Computes the primary key of the certificate using first 16 bytes of SHA-256 digest
+        """
+        return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link)
+
+    def merge(self, other: 'CommonCriteriaCert'):
+        """
+        Merges with other CC certificate. Assuming they come from different sources, e.g., csv and html.
+        Assuming that html source has better protection profiles, they overwrite CSV info
+        On other values (apart from maintainances, see TODO below) the sanity checks are made.
+        """
+        if self != other:
+            logging.warning(f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}')
+
+        for att, val in vars(self).items():
+            if not val:
+                setattr(self, att, getattr(other, att))
+            elif self.src == 'csv' and other.src == 'html' and att == 'protection_profiles':
+                setattr(self, att, getattr(other, att))
+            elif att == 'maintainances':
+                # TODO Fix me: This is a simplification. Basically take the longer list of maintainances as a ground truth.
+                if len(getattr(self, att)) < len(getattr(other, att)):
+                    setattr(self, att, getattr(other, att))
+            elif att == 'src':
+                pass  # This is expected
+            else:
+                if getattr(self, att) != getattr(other, att):
+                    logging.warning(f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}')
+        if self.src != other.src:
+            self.src = self.src + ' + ' + other.src
+
+    def to_dict(self) -> dict:
+        return self.__dict__
+
+    @classmethod
+    def from_dict(cls, dct: dict) -> 'CommonCriteriaCert':
+        # TODO: Implement me
         pass
 
     @classmethod
-    def from_dict(cls, dct):
-        return FIPSCertificate()
+    def from_html_row(cls, row: Tag, category: str) -> 'CommonCriteriaCert':
+        """
+        Creates a CC certificate from html row
+        """
+        def get_name(cell: Tag) -> str:
+            return list(cell.stripped_strings)[0]
+
+        def get_manufacturer(cell: Tag) -> Union[str, None]:
+            if lst := list(cell.stripped_strings):
+                return lst[0]
+            else:
+                return None
+
+        def get_scheme(cell: Tag) -> str:
+            return list(cell.stripped_strings)[0]
+
+        def get_security_level(cell: Tag) -> set:
+            return set(cell.stripped_strings)
+
+        def get_manufacturer_web(cell: Tag) -> Union[str, None]:
+            for link in cell.find_all('a'):
+                if link is not None and link.get('title') == 'Vendor\'s web site' and link.get('href') != 'http://':
+                   return link.get('href')
+            return None
+
+        def get_protection_profiles(cell: Tag) -> set:
+            protection_profiles = set()
+            for link in list(cell.find_all('a')):
+                if link.get('href') is not None and '/ppfiles/' in link.get('href'):
+                    protection_profiles.add(CommonCriteriaCert.ProtectionProfile(str(link.contents[0]), CommonCriteriaCert.cc_url + link.get('href')))
+            return protection_profiles
+
+        def get_date(cell: Tag) -> date:
+            text = cell.get_text()
+            extracted_date = datetime.strptime(text, '%Y-%m-%d').date() if text else None
+            return extracted_date
+
+        def get_report_st_links(cell: Tag) -> (str, str):
+            links = cell.find_all('a')
+            # TODO: Exception checks
+            assert links[1].get('title').startswith('Certification Report')
+            assert links[2].get('title').startswith('Security Target')
+
+            report_link = CommonCriteriaCert.cc_url + links[1].get('href')
+            security_target_link = CommonCriteriaCert.cc_url + links[2].get('href')
+
+            return report_link, security_target_link
+
+        def get_cert_link(cell: Tag) -> Union[str, None]:
+            links = cell.find_all('a')
+            return CommonCriteriaCert.cc_url + links[0].get('href') if links else None
+
+        def get_maintainance_div(cell: Tag) -> Union[Tag, None]:
+            divs = cell.find_all('div')
+            for d in divs:
+                if d.find('div') and d.stripped_strings and list(d.stripped_strings)[0] == 'Maintenance Report(s)':
+                    return d
+            return None
+
+        def get_maintainance_updates(main_div: Tag) -> set:
+            possible_updates = list(main_div.find_all('li'))
+            maintainance_updates = set()
+            for u in possible_updates:
+                text = list(u.stripped_strings)[0]
+                main_date = datetime.strptime(text.split(' ')[0], '%Y-%m-%d').date() if text else None
+                main_title = text.split('– ')[1]
+                main_report_link = None
+                main_st_link = None
+                links = u.find_all('a')
+                for l in links:
+                    if l.get('title').startswith('Maintenance Report:'):
+                        main_report_link = CommonCriteriaCert.cc_url + l.get('href')
+                    elif l.get('title').startswith('Maintenance ST'):
+                        main_st_link = CommonCriteriaCert.cc_url + l.get('href')
+                    else:
+                        logging.error('Unknown link in Maintenance part!')
+                maintainance_updates.add(CommonCriteriaCert.MaintainanceReport(main_date, main_title, main_report_link, main_st_link))
+            return maintainance_updates
+
+        cells = list(row.find_all('td'))
+        if len(cells) != 7:
+            logging.error('Unexpected number of cells in CC html row.')
+            raise
+
+        name = get_name(cells[0])
+        manufacturer = get_manufacturer(cells[1])
+        manufacturer_web = get_manufacturer_web(cells[1])
+        scheme = get_scheme(cells[6])
+        security_level = get_security_level(cells[5])
+        protection_profiles = get_protection_profiles(cells[0])
+        not_valid_before = get_date(cells[3])
+        not_valid_after = get_date(cells[4])
+        report_link, st_link = get_report_st_links(cells[0])
+        cert_link = get_cert_link(cells[2])
+
+        maintainance_div = get_maintainance_div(cells[0])
+        maintainances = get_maintainance_updates(maintainance_div) if maintainance_div else set()
+
+        crt = CommonCriteriaCert(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link, st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances)
+
+        return crt
diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index 8d2e0657..76b29390 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -1,10 +1,13 @@
 from enum import Enum
 
 
+
+
 class CertFramework(Enum):
     CC = 'Common Criteria'
     FIPS = 'FIPS'
 
+
 TAG_MATCH_COUNTER = 'count'
 TAG_MATCH_MATCHES = 'matches'
 
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index b77eb962..c430929b 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -1,46 +1,62 @@
-from .certificate import CommonCriteriaCert
-from . import constants
-from datetime import datetime
-from .meta_parser import CCMetaParser
-from .certificate import CommonCriteriaCert
-from . import constants
+from datetime import datetime, date
+from .certificate import CommonCriteriaCert, Certificate
+from abc import ABC, abstractmethod
+from . import helpers as helpers
+from pathlib import Path
+import shutil
+import pandas as pd
+from bs4 import BeautifulSoup
+import locale
+import logging
+from typing import Dict
+import json
 
 
-class Dataset:
-    def __init__(self, certs, framework, data_dir, name='dataset name', description='dataset_description'):
+class DatasetJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Certificate):
+            return obj.to_dict()
+        if isinstance(obj, set):
+            return list(obj)
+        if isinstance(obj, date):
+            return str(obj)
+        if isinstance(obj, CommonCriteriaCert.ProtectionProfile):
+            return obj.to_dict()
+        if isinstance(obj, CommonCriteriaCert.MaintainanceReport):
+            return obj.to_dict()
+        if isinstance(obj, Dataset):
+            return list(obj.certs.values())
+
+        return super().default(obj)
+
+
+class Dataset(ABC):
+    def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name', description: str = 'dataset_description'):
         self.certs = certs
-        self.framework = framework
-        self.data_dir = data_dir
+        self.root_dir = root_dir
 
         self.timestamp = datetime.now()
         self.sha256_digest = 'not implemented'
         self.name = name
+        self.description = description
 
-        self.sha256_digest = 'not implemented'  # TODO: Implement as hash of all certs
-        self.name = name  # TODO: Allow for naming of the datasets
-        self.description = description  # Allow for descriptions of the dataset
-
-    # The idea is to iterate over dataset as: `for cert in Dataset: ...
     def __iter__(self):
         for cert in self.certs.values():
             yield cert
 
-    # The idea is to be able to access certificates by calling Dataset[cert_hash]
-    def __getitem__(self, item):
+    def __getitem__(self, item: str) -> 'Certificate':
         return self.certs.__getitem__(item.lower())
 
-    # Same as above, but setting instead of getting
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: 'Certificate'):
         self.certs.__setitem__(key.lower(), value)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.certs)
 
-    # Not sure if we want such behaviour
-    def __eq__(self, other):
+    def __eq__(self, other: 'Dataset') -> bool:
         return self.certs == other.certs
 
-    def __str__(self):
+    def __str__(self) -> str:
         return 'Not implemented'
 
     def to_json(self):
@@ -60,35 +76,199 @@ class Dataset:
     def from_csv(cls):
         pass
 
-    @classmethod
-    def from_dataframe(cls, df):
+    def dump_to_json(self):
         pass
 
-    def merge(self, other):
-        return self
-
-    def dump_to_json(self):
+    @abstractmethod
+    def get_certs_from_web(self):
         pass
 
-    @classmethod
-    def init_from_meta(cls, data_dir, name, description):
+    def merge(self, certs: Dict[str, 'Certificate']):
         pass
 
 
 class CCDataset(Dataset):
-    @classmethod
-    def init_from_meta(cls, data_dir, name, description):
-        parser = CCMetaParser()
-        records = parser.parse_meta()
-        certs = {}
+    @property
+    def web_dir(self) -> Path:
+        return self.root_dir / 'web'
+
+    html_products = {
+        'cc_products_active.html': 'https://www.commoncriteriaportal.org/products/',
+        'cc_products_archived.html': 'https://www.commoncriteriaportal.org/products/index.cfm?archived=1',
+    }
+    html_labs = {'cc_labs.html': 'https://www.commoncriteriaportal.org/labs'}
+    csv_products = {
+        'cc_products_active.csv': 'https://www.commoncriteriaportal.org/products/certified_products.csv',
+        'cc_products_archived.csv': 'https://www.commoncriteriaportal.org/products/certified_products-archived.csv',
+    }
+    html_pp = {
+        'cc_pp_active.html': 'https://www.commoncriteriaportal.org/pps/',
+        'cc_pp_collaborative.html': 'https://www.commoncriteriaportal.org/pps/collaborativePP.cfm?cpp=1',
+        'cc_pp_archived.html': 'https://www.commoncriteriaportal.org/pps/index.cfm?archived=1',
+    }
+    csv_pp = {
+        'cc_pp_active.csv': 'https://www.commoncriteriaportal.org/pps/pps.csv',
+        'cc_pp_archived.csv': 'https://www.commoncriteriaportal.org/pps/pps-archived.csv'
+    }
+
+    def merge_certs(self, certs: Dict[str, 'CommonCriteriaCert']):
+        """
+        Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates
+        """
+        will_be_added = {}
+        n_merged = 0
+        for crt in certs.values():
+            if crt not in self:
+                will_be_added[crt.dgst] = crt
+            else:
+                self[crt.dgst].merge(crt)
+                n_merged += 1
+
+        self.certs.update(will_be_added)
+        logging.info(f'Added {len(will_be_added)} new and merged further {n_merged} certificates to the dataset.')
+
+    def get_certs_from_web(self, keep_metadata: bool = True):
+        """
+        Downloads all metadata about certificates from CSV and HTML sources
+        """
+        self.web_dir.mkdir(parents=True, exist_ok=True)
+
+        html_items = [(x, self.web_dir / y) for y, x in self.html_products.items()]
+        csv_items = [(x, self.web_dir / y) for y, x in self.csv_products.items()]
+        helpers.download_parallel(html_items, num_threads=8)
+        helpers.download_parallel(csv_items, num_threads=8)
+
+        logging.info('Adding CSV certificates to CommonCriteria dataset.')
+        csv_certs = self.get_all_certs_from_csv()
+        self.merge_certs(csv_certs)
+
+        # TODO: Someway along the way, 3 certificates get lost. Investigate and fix.
+        logging.info('Adding HTML certificates to CommonCriteria dataset.')
+        html_certs = self.get_all_certs_from_html()
+        self.merge_certs(html_certs)
+
+        logging.info(f'The resulting dataset has {len(self)} certificates.')
 
-        for r in records:
-            cert = CommonCriteriaCert.from_dict(r)
-            certs[cert.sha256] = cert
+        if not keep_metadata:
+            shutil.rmtree(self.web_dir)
 
-        return CCDataset(certs, constants.CertFramework.CC, data_dir, name, description)
+    def get_all_certs_from_csv(self) -> Dict[str, 'CommonCriteriaCert']:
+        """
+        Creates dictionary of new certificates from csv sources.
+        """
+        new_certs = {}
+        for file in self.csv_products:
+            partial_certs = self.parse_single_csv(self.web_dir / file)
+            logging.info(f'Parsed {len(partial_certs)} certificates from: {file}')
+            new_certs.update(partial_certs)
+        return new_certs
 
+    @staticmethod
+    def parse_single_csv(file: Path) -> Dict[str, 'CommonCriteriaCert']:
+        """
+        Using pandas, this parses a single CSV file.
+        """
+        def get_primary_key_str(row):
+            prim_key = row['category'] + row['cert_name'] + row['report_link']
+            return prim_key
 
-class FIPSDataset(Dataset):
-    pass
+        csv_header = ['category', 'cert_name', 'manufacturer', 'scheme', 'security_level', 'protection_profiles',
+                       'not_valid_before', 'not_valid_after', 'report_link', 'st_link', 'maintainance_date',
+                       'maintainance_title', 'maintainance_report_link', 'maintainance_st_link']
+
+        df = pd.read_csv(file, engine='python', encoding='windows-1250')
+        df = df.rename(columns={x: y for (x, y) in zip(list(df.columns), csv_header)})
+
+        df['is_maintainance'] = ~df.maintainance_title.isnull()
+        df = df.fillna(value='')
+
+        df[['not_valid_before', 'not_valid_after', 'maintainance_date']] = df[['not_valid_before', 'not_valid_after', 'maintainance_date']].apply(pd.to_datetime)
+
+        df['dgst'] = df.apply(lambda row: helpers.get_first_16_bytes_sha256(get_primary_key_str(row)), axis=1)
+        df_base = df.loc[df.is_maintainance == False].copy()
+        df_main = df.loc[df.is_maintainance == True].copy()
+
+        n_all = len(df_base)
+        n_deduplicated = len(df_base.drop_duplicates(subset=['dgst']))
+        logging.warning(f'The CSV {file} contains {n_all - n_deduplicated} duplicates by the primary key.')
+
+        df_base = df_base.drop_duplicates(subset=['dgst'])
+        df_main = df_main.drop_duplicates()
+
+        profiles = {x.dgst: set([CommonCriteriaCert.ProtectionProfile(y, None) for y in helpers.sanitize_protection_profiles(x.protection_profiles)]) for x in df_base.itertuples()}
+        updates = {x.dgst: set() for x in df_base.itertuples()}
+        for x in df_main.itertuples():
+            updates[x.dgst].add(CommonCriteriaCert.MaintainanceReport(x.maintainance_date.date(), x.maintainance_title, x.maintainance_report_link, x.maintainance_st_link))
+
+        certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level, x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv', None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None)) for x in df_base.itertuples()}
+        return certs
+
+    def get_all_certs_from_html(self) -> Dict[str, 'CommonCriteriaCert']:
+        """
+        Prepares dictionary of certificates from all html files.
+        """
+        new_certs = {}
+        for file in self.html_products:
+            partial_certs = self.parse_single_html(self.web_dir / file)
+            logging.info(f'Parsed {len(partial_certs)} certificates from: {file}')
+            new_certs.update(partial_certs)
+        return new_certs
+
+    @staticmethod
+    def parse_single_html(file: Path) -> Dict[str, 'CommonCriteriaCert']:
+        """
+        Prepares a dictionary of certificates from a single html file.
+        """
+        def get_timestamp_from_footer(footer):
+            locale.setlocale(locale.LC_ALL, 'en_US')
+            footer_text = list(footer.stripped_strings)[0]
+            date_string = footer_text.split(',')[1:3]
+            time_string = footer_text.split(',')[3].split(' at ')[1]
+            formatted_datetime = date_string[0] + date_string[1] + ' ' + time_string
+            return datetime.strptime(formatted_datetime, ' %B %d %Y %I:%M %p')
+
+        def parse_table(soup: BeautifulSoup, table_id: str, category_string: str) -> Dict[str, 'CommonCriteriaCert']:
+            tables = soup.find_all('table', id=table_id)
+            assert len(tables) == 1
+            table = tables[0]
+            rows = list(table.find_all('tr'))
+            header, footer, body = rows[0], rows[1], rows[2:]
+
+            # TODO: It's possible to obtain timestamp of the moment when the list was generated. It's identical for each table and should thus only be obtained once. Not necessarily in each table
+            # timestamp = get_timestamp_from_footer(footer)
+
+            # TODO: Do we have use for number of expected certs? We get rid of duplicites, so no use for assert expected == actual
+            # caption_str = str(table.findAll('caption'))
+            # n_expected_certs = int(caption_str.split(category_string + ' – ')[1].split(' Certified Products')[0])
+            table_certs = {x.dgst: x for x in [CommonCriteriaCert.from_html_row(row, category_string) for row in body]}
+
+            return table_certs
+
+        cc_cat_abbreviations = ['AC', 'BP', 'DP', 'DB', 'DD', 'IC', 'KM',
+                                'MD', 'MF', 'NS', 'OS', 'OD', 'DG', 'TC']
+        cc_table_ids = ['tbl' + x for x in cc_cat_abbreviations]
+        cc_categories = ['Access Control Devices and Systems',
+                         'Boundary Protection Devices and Systems',
+                         'Data Protection',
+                         'Databases',
+                         'Detection Devices and Systems',
+                         'ICs, Smart Cards and Smart Card-Related Devices and Systems',
+                         'Key Management Systems',
+                         'Mobility',
+                         'Multi-Function Devices',
+                         'Network and Network-Related Devices and Systems',
+                         'Operating Systems',
+                         'Other Devices and Systems',
+                         'Products for Digital Signatures',
+                         'Trusted Computing'
+                         ]
+        cat_dict = {x: y for (x, y) in zip(cc_table_ids, cc_categories)}
+
+        with open(file, 'r') as handle:
+            soup = BeautifulSoup(handle, 'html.parser')
+
+        certs = {}
+        for key, val in cat_dict.items():
+            certs.update(parse_table(soup, key, val))
 
+        return certs
diff --git a/sec_certs/dataset_loader.py b/sec_certs/dataset_loader.py
deleted file mode 100644
index e6094336..00000000
--- a/sec_certs/dataset_loader.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from abc import ABC, abstractmethod
-from .dataset import Dataset
-import pandas as pd
-
-
-class CertLoader(ABC):
-    def __init__(self, download_pdfs, meta_dir, pdf_dir):
-        self.df = pd.DataFrame()
-        self.download_pdfs = download_pdfs
-        self.meta_dir = meta_dir
-        self.pdf_dir = pdf_dir
-        self.certs = {}
-
-    @abstractmethod
-    def load(self):
-        pass
-
-
-class CCCertLoader(CertLoader):
-
-    def load_csv(self):
-        pass
-
-    def load_html(self):
-        pass
-
-    def load(self):
-        self.load_csv()
-        self.load_html()
-        return self.certs
-
-
-class FIPSCertLoader(CertLoader):
-    def load_csv(self):
-        pass
-
-    def load_html(self):
-        pass
-
-    def load(self):
-        self.load_csv()
-        self.load_html()
-        return self.certs
-
diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py
index 8913d91f..35ae843d 100644
--- a/sec_certs/helpers.py
+++ b/sec_certs/helpers.py
@@ -1 +1,80 @@
-#TBA
\ No newline at end of file
+from typing import Sequence, Tuple
+import requests
+from multiprocessing.pool import ThreadPool
+from pathlib import Path
+from tqdm import tqdm
+import hashlib
+import html
+from typing import Union
+from datetime import date
+import numpy as np
+import pandas as pd
+
+
+def download_file(url: str, output: Path) -> int:
+    r = requests.get(url, allow_redirects=True)
+    with output.open('wb') as f:
+        f.write(r.content)
+    return r.status_code
+
+
+def download_parallel(items: Sequence[Tuple[str, Path]], num_threads: int) -> Sequence[Tuple[str, int]]:
+    def download(url_output):
+        url, output = url_output
+        return url, download_file(url, output)
+
+    pool = ThreadPool(num_threads)
+    responses = []
+    with tqdm(total=len(items)) as progress:
+        for response in pool.imap(download, items):
+            progress.update(1)
+            responses.append(response)
+    pool.close()
+    pool.join()
+    return responses
+
+
+def get_first_16_bytes_sha256(string: str) -> str:
+    return hashlib.sha256(string.encode('utf-8')).hexdigest()[:16]
+
+
+def sanitize_link(record: str) -> Union[str, None]:
+    if not record:
+        return None
+    return record.replace(':443', '').replace(' ', '%20')
+
+
+def sanitize_date(record: Union[pd.Timestamp, date, np.datetime64]) -> Union[date, None]:
+    if pd.isnull(record):
+        return None
+    elif isinstance(record, pd.Timestamp):
+        return record.date()
+    else:
+        return record
+
+
+def sanitize_string(record: str) -> Union[str, None]:
+    if not record:
+        return None
+    else:
+        # TODO: There is a certificate with name 'ATMEL Secure Microcontroller AT90SC12872RCFT &#x2f; AT90SC12836RCFT rev. I &amp;&#x23;38&#x3b; J' that has to be unescaped twice
+        return html.unescape(html.unescape(record)).replace('\r\n', ' ').replace('\n', '')
+
+
+def sanitize_security_levels(record: Union[str, set]) -> set:
+    if isinstance(record, str):
+        record = set(record.split(','))
+
+    if 'PP\xa0Compliant' in record:
+        record.remove('PP\xa0Compliant')
+
+    if 'None' in record:
+        record.remove('None')
+
+    return record
+
+
+def sanitize_protection_profiles(record: str) -> list:
+    if not record:
+        return []
+    return record.split(',')
\ No newline at end of file
diff --git a/sec_certs/meta_parser.py b/sec_certs/meta_parser.py
deleted file mode 100644
index bd72bae7..00000000
--- a/sec_certs/meta_parser.py
+++ /dev/null
@@ -1,28 +0,0 @@
-class MetaParser:
-    def parse_certs_from_html(self):
-        raise NotImplementedError('Meant to be provided by child classes')
-
-    def parse_certs_from_csv(self):
-        raise NotImplementedError('Meant to be provided by child classes')
-
-
-class CCMetaParser(MetaParser):
-    def __init__(self):
-        self.records = {}
-
-    def parse_meta(self):
-        self.parse_certs_from_csv()
-        self.parse_certs_from_html()
-
-        return self.records
-
-    def parse_certs_from_csv(self):
-        pass
-
-    def parse_certs_from_html(self):
-        pass
-
-
-class FIPSMetaParser(MetaParser):
-    def __init__(self):
-        pass
\ No newline at end of file