refactor folder strucure

9 files changed, 1249 insertions, 1201 deletions

diff --git a/sec_certs/certificate/certificate.py b/sec_certs/certificate/certificate.py
new file mode 100644
index 00000000..dddac2b7
--- /dev/null
+++ b/sec_certs/certificate/certificate.py
@@ -0,0 +1,51 @@
+import logging
+from pathlib import Path
+import copy
+import json
+
+from abc import ABC, abstractmethod
+from typing import Union, TypeVar, Type
+
+from sec_certs.serialization import CustomJSONDecoder, CustomJSONEncoder
+
+logger = logging.getLogger(__name__)
+
+
+class Certificate(ABC):
+    T = TypeVar('T', bound='Certificate')
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def __repr__(self) -> str:
+        return str(self.to_dict())
+
+    def __str__(self) -> str:
+        return 'Not implemented'
+
+    @property
+    @abstractmethod
+    def dgst(self):
+        raise NotImplementedError('Not meant to be implemented')
+
+    def __eq__(self, other: 'Certificate') -> bool:
+        return self.dgst == other.dgst
+
+    def to_dict(self):
+        return {**{'dgst': self.dgst}, **copy.deepcopy(self.__dict__)}
+
+    @classmethod
+    def from_dict(cls: Type[T], dct: dict) -> T:
+        dct.pop('dgst')
+        return cls(*(tuple(dct.values())))
+
+    def to_json(self, output_path: Union[Path, str]):
+        with Path(output_path).open('w') as handle:
+            json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
+
+    @classmethod
+    def from_json(cls, input_path: Union[Path, str]):
+        with Path(input_path).open('r') as handle:
+            return json.load(handle, cls=CustomJSONDecoder)
+
+
diff --git a/sec_certs/certificate/common_criteria.py b/sec_certs/certificate/common_criteria.py
new file mode 100644
index 00000000..669e497d
--- /dev/null
+++ b/sec_certs/certificate/common_criteria.py
@@ -0,0 +1,551 @@
+import copy
+import itertools
+import re
+from dataclasses import dataclass, field
+from datetime import date, datetime
+from pathlib import Path
+from typing import Optional, List, Dict, Tuple, Union
+
+import requests
+from bs4 import Tag
+
+from sec_certs import helpers, constants as constants
+from sec_certs.certificate.certificate import Certificate, logger
+from sec_certs.dataset.cpe import CPE, CPEDataset
+from sec_certs.dataset.cve import CVE, CVEDataset
+from sec_certs.serialization import ComplexSerializableType
+
+
+class CommonCriteriaCert(Certificate, ComplexSerializableType):
+    cc_url = 'http://www.commoncriteriaportal.org'
+    empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/'
+
+    @dataclass(eq=True, frozen=True)
+    class MaintainanceReport(ComplexSerializableType):
+        """
+        Object for holding maintainance reports.
+        """
+        maintainance_date: date
+        maintainance_title: str
+        maintainance_report_link: str
+        maintainance_st_link: str
+
+        def __post_init__(self):
+            super().__setattr__('maintainance_report_link',
+                                helpers.sanitize_link(self.maintainance_report_link))
+            super().__setattr__('maintainance_st_link',
+                                helpers.sanitize_link(self.maintainance_st_link))
+            super().__setattr__('maintainance_title',
+                                helpers.sanitize_string(self.maintainance_title))
+            super().__setattr__('maintainance_date', helpers.sanitize_date(self.maintainance_date))
+
+        def to_dict(self):
+            return copy.deepcopy(self.__dict__)
+
+        @classmethod
+        def from_dict(cls, dct):
+            return cls(*tuple(dct.values()))
+
+        def __lt__(self, other):
+            return self.maintainance_date < other.maintainance_date
+
+    @dataclass(eq=True, frozen=True)
+    class ProtectionProfile(ComplexSerializableType):
+        """
+        Object for holding protection profiles.
+        """
+        pp_name: str
+        pp_link: Optional[str]
+
+        def __post_init__(self):
+            super().__setattr__('pp_name', helpers.sanitize_string(self.pp_name))
+            super().__setattr__('pp_link', helpers.sanitize_link(self.pp_link))
+
+        def to_dict(self):
+            return copy.deepcopy(self.__dict__)
+
+        @classmethod
+        def from_dict(cls, dct):
+            return cls(*tuple(dct.values()))
+
+        def __lt__(self, other):
+            return self.pp_name < other.pp_name
+
+    @dataclass(init=False)
+    class InternalState(ComplexSerializableType):
+        st_link_ok: bool
+        report_link_ok: bool
+        st_convert_ok: bool
+        report_convert_ok: bool
+        st_extract_ok: bool
+        report_extract_ok: bool
+        st_pdf_path: Path
+        report_pdf_path: Path
+        st_txt_path: Path
+        report_txt_path: Path
+        errors: Optional[List[str]]
+
+        def __init__(self, st_link_ok: bool = True, report_link_ok: bool = True,
+                     st_convert_ok: bool = True, report_convert_ok: bool = True,
+                     st_extract_ok: bool = True, report_extract_ok: bool = True,
+                     errors: Optional[List[str]] = None):
+            self.st_link_ok = st_link_ok
+            self.report_link_ok = report_link_ok
+            self.st_convert_ok = st_convert_ok
+            self.report_convert_ok = report_convert_ok
+            self.st_extract_ok = st_extract_ok
+            self.report_extract_ok = report_extract_ok
+
+            if errors is None:
+                self.errors = []
+            else:
+                self.errors = errors
+
+        def to_dict(self):
+            return {'st_link_ok': self.st_link_ok, 'report_link_ok': self.report_link_ok,
+                    'st_convert_ok': self.st_convert_ok, 'report_convert_ok': self.report_convert_ok,
+                    'st_extract_ok': self.st_extract_ok, 'report_extract_ok': self.report_extract_ok,
+                    'errors': self.errors}
+
+        @classmethod
+        def from_dict(cls, dct: Dict[str, bool]):
+            return cls(*tuple(dct.values()))
+
+    @dataclass(init=False)
+    class PdfData(ComplexSerializableType):
+        report_metadata: Dict[str, str]
+        st_metadata: Dict[str, str]
+        report_frontpage: Dict[str, str]
+        st_frontpage: Dict[str, str]
+        report_keywords: Dict[str, str]
+        st_keywords: Dict[str, str]
+
+        def __init__(self, report_metadata: Optional[Dict[str, str]] = None,
+                     st_metadata: Optional[Dict[str, str]] = None,
+                     report_frontpage: Optional[Dict[str, str]] = None, st_frontpage: Optional[Dict[str, str]] = None,
+                     report_keywords: Optional[Dict[str, str]] = None, st_keywords: Optional[Dict[str, str]] = None):
+            self.report_metadata = report_metadata
+            self.st_metadata = st_metadata
+            self.report_frontpage = report_frontpage
+            self.st_frontpage = st_frontpage
+            self.report_keywords = report_keywords
+            self.st_keywords = st_keywords
+
+        def to_dict(self):
+            return {'report_metadata': self.report_metadata, 'st_metadata': self.st_metadata,
+                    'report_frontpage': self.report_frontpage,
+                    'st_frontpage': self.st_frontpage, 'report_keywords': self.report_keywords,
+                    'st_keywords': self.st_keywords}
+
+        @classmethod
+        def from_dict(cls, dct: Dict[str, bool]):
+            return cls(*tuple(dct.values()))
+
+    @dataclass(init=False)
+    class Heuristics(ComplexSerializableType):
+        extracted_versions: List[str]
+        cpe_candidate_vendors: Optional[List[str]] = field(init=False)
+        cpe_matches: Optional[List[Tuple[float, CPE]]]
+        verified_cpe_matches: Optional[List[CPE]]
+        related_cves: Optional[List[str]]
+
+        def __init__(self,
+                     extracted_versions: Optional[List[str]] = None,
+                     cpe_matches: Optional[List[str]] = None,
+                     verified_cpe_matches: Optional[List[str]] = None,
+                     related_cves: Optional[List[CVE]] = None):
+            self.extracted_versions = extracted_versions
+            self.cpe_matches = cpe_matches
+            self.cpe_candidate_vendors = None
+            self.verified_cpe_matches = verified_cpe_matches
+            self.related_cves = related_cves
+
+        def to_dict(self):
+            return {'extracted_versions': self.extracted_versions, 'cpe_matches': self.cpe_matches, 'verified_cpe_matches': self.verified_cpe_matches, 'related_cves': self.related_cves}
+
+        @classmethod
+        def from_dict(cls, dct: Dict[str, str]):
+            return cls(*tuple(dct.values()))
+
+    pandas_columns = ['dgst', 'name', 'status', 'category', 'manufacturer', 'scheme', 'security_level',
+                      'not_valid_before', 'not_valid_after', 'report_link', 'st_link',
+                      'manufacturer_web', 'extracted_versions', 'cpe_matches', 'verified_cpe_matches',
+                      'related_cves']
+
+    def __init__(self, status: str, category: str, name: str, manufacturer: str, scheme: str,
+                 security_level: Union[str, set], not_valid_before: date,
+                 not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str],
+                 manufacturer_web: Optional[str],
+                 protection_profiles: set,
+                 maintainance_updates: set,
+                 state: Optional[InternalState],
+                 pdf_data: Optional[PdfData],
+                 heuristics: Optional[Heuristics]):
+        super().__init__()
+
+        self.status = status
+        self.category = category
+        self.name = helpers.sanitize_string(name)
+        self.manufacturer = helpers.sanitize_string(manufacturer)
+        self.scheme = scheme
+        self.security_level = helpers.sanitize_security_levels(security_level)
+        self.not_valid_before = helpers.sanitize_date(not_valid_before)
+        self.not_valid_after = helpers.sanitize_date(not_valid_after)
+        self.report_link = helpers.sanitize_link(report_link)
+        self.st_link = helpers.sanitize_link(st_link)
+        self.src = src
+        self.cert_link = helpers.sanitize_link(cert_link)
+        self.manufacturer_web = helpers.sanitize_link(manufacturer_web)
+        self.protection_profiles = protection_profiles
+        self.maintainance_updates = maintainance_updates
+
+        if state is None:
+            state = self.InternalState()
+        self.state = state
+
+        if pdf_data is None:
+            pdf_data = self.PdfData()
+        self.pdf_data = pdf_data
+
+        if heuristics is None:
+            heuristics = self.Heuristics()
+        self.heuristics = heuristics
+
+    @property
+    def dgst(self) -> str:
+        """
+        Computes the primary key of the certificate using first 16 bytes of SHA-256 digest
+        """
+        return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link)
+
+    def __str__(self):
+        return self.manufacturer + ' ' + self.name + ' dgst: ' + self.dgst
+
+    def to_pandas_tuple(self):
+        return self.dgst, self.name, self.status, self.category, self.manufacturer, self.scheme, self.security_level,\
+               self.not_valid_before, self.not_valid_after, self.report_link, self.st_link, self.manufacturer_web, \
+               self.heuristics.extracted_versions, self.heuristics.cpe_matches, self.heuristics.verified_cpe_matches, \
+               self.heuristics.related_cves
+
+
+    def merge(self, other: 'CommonCriteriaCert'):
+        """
+        Merges with other CC certificate. Assuming they come from different sources, e.g., csv and html.
+        Assuming that html source has better protection profiles, they overwrite CSV info
+        On other values (apart from maintainances, see TODO below) the sanity checks are made.
+        """
+        if self != other:
+            logger.warning(
+                f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}')
+
+        for att, val in vars(self).items():
+            if not val:
+                setattr(self, att, getattr(other, att))
+            elif self.src == 'csv' and other.src == 'html' and att == 'protection_profiles':
+                setattr(self, att, getattr(other, att))
+            elif self.src == 'csv' and other.src == 'html' and att == 'maintainance_updates':
+                # TODO Fix me: This is a simplification. At the moment html contains more reliable info
+                setattr(self, att, getattr(other, att))
+            elif att == 'src':
+                pass  # This is expected
+            elif att == 'state':
+                setattr(self, att, getattr(other, att))
+            else:
+                if getattr(self, att) != getattr(other, att):
+                    logger.warning(
+                        f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}')
+        if self.src != other.src:
+            self.src = self.src + ' + ' + other.src
+
+    @classmethod
+    def from_dict(cls, dct: Dict) -> 'CommonCriteriaCert':
+        new_dct = dct.copy()
+        new_dct['maintainance_updates'] = set(dct['maintainance_updates'])
+        new_dct['protection_profiles'] = set(dct['protection_profiles'])
+        return super(cls, CommonCriteriaCert).from_dict(new_dct)
+
+    @classmethod
+    def from_html_row(cls, row: Tag, status: str, category: str) -> 'CommonCriteriaCert':
+        """
+        Creates a CC certificate from html row
+        """
+
+        def _get_name(cell: Tag) -> str:
+            return list(cell.stripped_strings)[0]
+
+        def _get_manufacturer(cell: Tag) -> Optional[str]:
+            if lst := list(cell.stripped_strings):
+                return lst[0]
+            else:
+                return None
+
+        def _get_scheme(cell: Tag) -> str:
+            return list(cell.stripped_strings)[0]
+
+        def _get_security_level(cell: Tag) -> set:
+            return set(cell.stripped_strings)
+
+        def _get_manufacturer_web(cell: Tag) -> Optional[str]:
+            for link in cell.find_all('a'):
+                if link is not None and link.get('title') == 'Vendor\'s web site' and link.get('href') != 'http://':
+                    return link.get('href')
+            return None
+
+        def _get_protection_profiles(cell: Tag) -> set:
+            protection_profiles = set()
+            for link in list(cell.find_all('a')):
+                if link.get('href') is not None and '/ppfiles/' in link.get('href'):
+                    protection_profiles.add(CommonCriteriaCert.ProtectionProfile(str(link.contents[0]),
+                                                                                 CommonCriteriaCert.cc_url + link.get(
+                                                                                     'href')))
+            return protection_profiles
+
+        def _get_date(cell: Tag) -> date:
+            text = cell.get_text()
+            extracted_date = datetime.strptime(
+                text, '%Y-%m-%d').date() if text else None
+            return extracted_date
+
+        def _get_report_st_links(cell: Tag) -> (str, str):
+            links = cell.find_all('a')
+            # TODO: Exception checks
+            assert links[1].get('title').startswith('Certification Report')
+            assert links[2].get('title').startswith('Security Target')
+
+            report_link = CommonCriteriaCert.cc_url + links[1].get('href')
+            security_target_link = CommonCriteriaCert.cc_url + \
+                                   links[2].get('href')
+
+            return report_link, security_target_link
+
+        def _get_cert_link(cell: Tag) -> Optional[str]:
+            links = cell.find_all('a')
+            return CommonCriteriaCert.cc_url + links[0].get('href') if links else None
+
+        def _get_maintainance_div(cell: Tag) -> Optional[Tag]:
+            divs = cell.find_all('div')
+            for d in divs:
+                if d.find('div') and d.stripped_strings and list(d.stripped_strings)[0] == 'Maintenance Report(s)':
+                    return d
+            return None
+
+        def _get_maintainance_updates(main_div: Tag) -> set:
+            possible_updates = list(main_div.find_all('li'))
+            maintainance_updates = set()
+            for u in possible_updates:
+                text = list(u.stripped_strings)[0]
+                main_date = datetime.strptime(text.split(
+                    ' ')[0], '%Y-%m-%d').date() if text else None
+                main_title = text.split('– ')[1]
+                main_report_link = None
+                main_st_link = None
+                links = u.find_all('a')
+                for l in links:
+                    if l.get('title').startswith('Maintenance Report:'):
+                        main_report_link = CommonCriteriaCert.cc_url + \
+                                           l.get('href')
+                    elif l.get('title').startswith('Maintenance ST'):
+                        main_st_link = CommonCriteriaCert.cc_url + \
+                                       l.get('href')
+                    else:
+                        logger.error('Unknown link in Maintenance part!')
+                maintainance_updates.add(
+                    CommonCriteriaCert.MaintainanceReport(main_date, main_title, main_report_link, main_st_link))
+            return maintainance_updates
+
+        cells = list(row.find_all('td'))
+        if len(cells) != 7:
+            logger.error('Unexpected number of cells in CC html row.')
+            raise
+
+        name = _get_name(cells[0])
+        manufacturer = _get_manufacturer(cells[1])
+        manufacturer_web = _get_manufacturer_web(cells[1])
+        scheme = _get_scheme(cells[6])
+        security_level = _get_security_level(cells[5])
+        protection_profiles = _get_protection_profiles(cells[0])
+        not_valid_before = _get_date(cells[3])
+        not_valid_after = _get_date(cells[4])
+        report_link, st_link = _get_report_st_links(cells[0])
+        cert_link = _get_cert_link(cells[2])
+
+        maintainance_div = _get_maintainance_div(cells[0])
+        maintainances = _get_maintainance_updates(
+            maintainance_div) if maintainance_div else set()
+
+        return cls(status, category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after,
+                   report_link,
+                   st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None, None, None)
+
+    def set_local_paths(self,
+                        report_pdf_dir: Optional[Union[str, Path]],
+                        st_pdf_dir: Optional[Union[str, Path]],
+                        report_txt_dir: Optional[Union[str, Path]],
+                        st_txt_dir: Optional[Union[str, Path]]):
+        if report_pdf_dir is not None:
+            self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + '.pdf')
+        if st_pdf_dir is not None:
+            self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + '.pdf')
+        if report_txt_dir is not None:
+            self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + '.txt')
+        if st_txt_dir is not None:
+            self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + '.txt')
+
+    @property
+    def best_cpe_match(self):
+        clean = [x for x in self.cpe_matching if len(x[0]) > 5]
+        cpe_match_ranking = [x[1] for x in clean]
+        argmax = cpe_match_ranking.index(max(cpe_match_ranking))
+        return clean[argmax]
+
+    @staticmethod
+    def download_pdf_report(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path)
+        if exit_code != requests.codes.ok:
+            error_msg = f'failed to download report from {cert.report_link}, code: {exit_code}'
+            logger.error(f'Cert dgst: {cert.dgst} ' + error_msg)
+            cert.state.report_link_ok = False
+            cert.state.errors.append(error_msg)
+        return cert
+
+    @staticmethod
+    def download_pdf_target(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path)
+        if exit_code != requests.codes.ok:
+            error_msg = f'failed to download ST from {cert.report_link}, code: {exit_code}'
+            logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
+            cert.state.st_link_ok = False
+            cert.state.errors.append(error_msg)
+        return cert
+
+    def path_is_corrupted(self, local_path):
+        return not local_path.exists() or local_path.stat().st_size < constants.MIN_CORRECT_CERT_SIZE
+
+    @staticmethod
+    def convert_report_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path, ['-raw'])
+        if exit_code != constants.RETURNCODE_OK:
+            error_msg = 'failed to convert report pdf->txt'
+            logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
+            cert.state.report_convert_ok = False
+            cert.state.errors.append(error_msg)
+        return cert
+
+    @staticmethod
+    def convert_target_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path, ['-raw'])
+        if exit_code != constants.RETURNCODE_OK:
+            error_msg = 'failed to convert security target pdf->txt'
+            logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
+            cert.state.st_convert_ok = False
+            cert.state.errors.append(error_msg)
+        return cert
+
+    @staticmethod
+    def extract_st_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        response, cert.pdf_data.st_metadata = helpers.extract_pdf_metadata(cert.state.st_pdf_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.st_extract_ok = False
+            cert.state.errors.append(response)
+        return cert
+
+    @staticmethod
+    def extract_report_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        response, cert.pdf_data.report_metadata = helpers.extract_pdf_metadata(cert.state.report_pdf_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.report_extract_ok = False
+            cert.state.errors.append(response)
+        return cert
+
+    @staticmethod
+    def extract_st_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        cert.pdf_data.st_frontpage = dict()
+
+        response_anssi, cert.pdf_data.st_frontpage['anssi'] = helpers.search_only_headers_anssi(cert.state.st_txt_path)
+        response_bsi, cert.pdf_data.st_frontpage['bsi'] = helpers.search_only_headers_bsi(cert.state.st_txt_path)
+
+        if response_anssi != constants.RETURNCODE_OK:
+            cert.state.st_extract_ok = False
+            cert.state.errors.append(response_anssi)
+        if response_bsi != constants.RETURNCODE_OK:
+            cert.state.st_extract_ok = False
+            cert.state.errors.append(response_bsi)
+
+        return cert
+
+    @staticmethod
+    def extract_report_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        cert.pdf_data.report_frontpage = dict()
+        response_bsi, cert.pdf_data.report_frontpage['bsi'] = helpers.search_only_headers_bsi(
+            cert.state.report_txt_path)
+        response_anssi, cert.pdf_data.report_frontpage['anssi'] = helpers.search_only_headers_anssi(
+            cert.state.report_txt_path)
+
+        if response_anssi != constants.RETURNCODE_OK:
+            cert.state.report_extract_ok = False
+            cert.state.errors.append(response_anssi)
+        if response_bsi != constants.RETURNCODE_OK:
+            cert.state.report_extract_ok = False
+            cert.state.errors.append(response_bsi)
+
+        return cert
+
+    @staticmethod
+    def extract_report_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        response, cert.pdf_data.report_keywords = helpers.extract_keywords(cert.state.report_txt_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.report_extract_ok = False
+        return cert
+
+    @staticmethod
+    def extract_st_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        response, cert.pdf_data.st_keywords = helpers.extract_keywords(cert.state.st_txt_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.st_extract_ok = False
+            cert.state.errors.append(response)
+        return cert
+
+    def compute_heuristics_version(self):
+        """
+        Will extract possible versions from the name
+        """
+        at_least_something = r'(\b(\d)+\b)'
+        just_numbers = r'(\d{1,5})(\.\d{1,5})'
+
+        without_version = r'(' + just_numbers + r'+)'
+        long_version = r'(' + r'(\bversion)\s*' + just_numbers + r'+)'
+        short_version = r'(' + r'\bv\s*' + just_numbers + r'+)'
+        full_regex_string = r'|'.join([without_version, short_version, long_version])
+        normalizer = r'(\d+\.*)+'
+
+        matched_strings = set([max(x, key=len) for x in re.findall(full_regex_string, self.name, re.IGNORECASE)])
+        if not matched_strings:
+            matched_strings = set([max(x, key=len) for x in re.findall(at_least_something, self.name, re.IGNORECASE)])
+
+        if matched_strings:
+            self.heuristics.extracted_versions = [re.search(normalizer, x).group() for x in matched_strings]
+        else:
+            self.heuristics.extracted_versions = ['-']
+
+    def compute_heuristics_cpe_vendors(self, cpe_dataset: CPEDataset):
+        """
+        With the help of the CPE dataset, will find CPE vendors that could match the given certificate vendor
+        """
+        self.heuristics.cpe_candidate_vendors = cpe_dataset.get_candidate_list_of_vendors(self.manufacturer)
+
+    def compute_heuristics_cpe_match(self, cpe_dataset: CPEDataset):
+        self.compute_heuristics_cpe_vendors(cpe_dataset)
+        self.heuristics.cpe_matches = cpe_dataset.get_cpe_matches(self.name,
+                                                                  self.heuristics.cpe_candidate_vendors,
+                                                                  self.heuristics.extracted_versions,
+                                                                  n_max_matches=constants.CPE_MAX_MATCHES,
+                                                                  threshold=constants.CPE_MATCHING_THRESHOLD)
+
+    def compute_heuristics_related_cves(self, cve_dataset: CVEDataset):
+        if self.heuristics.verified_cpe_matches:
+            related_cves = [cve_dataset.get_cves_for_cpe(x.uri) for x in self.heuristics.verified_cpe_matches]
+            related_cves = list(filter(lambda x: x is not None, related_cves))
+            if related_cves:
+                self.heuristics.related_cves = list(itertools.chain.from_iterable(related_cves))
+        else:
+            self.heuristics.related_cves = None
\ No newline at end of file
diff --git a/sec_certs/certificate.py b/sec_certs/certificate/fips.py
index b873ac37..3a37e344 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate/fips.py
@@ -1,70 +1,22 @@
+import copy
 import re
-from datetime import datetime, date
-from dataclasses import dataclass, field
-import logging
+from dataclasses import dataclass
+from datetime import datetime
 from pathlib import Path
-import os
-import copy
-import json
+from typing import ClassVar, Dict, Optional, Union, List, Tuple, Set
+
 import requests
+from bs4 import Tag, NavigableString, BeautifulSoup
 from dateutil import parser
-import itertools
-
-from abc import ABC, abstractmethod
-from bs4 import Tag, BeautifulSoup, NavigableString
-from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type, Tuple, Pattern, Set
-
 from tabula import read_pdf
 
-from sec_certs import helpers, extract_certificates, dataset
-from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
-import sec_certs.constants as constants
-from sec_certs.extract_certificates import load_cert_file, normalize_match_string, save_modified_cert_file, REGEXEC_SEP, \
-    LINE_SEPARATOR, APPEND_DETAILED_MATCH_MATCHES
-from sec_certs.cert_rules import fips_rules, fips_common_rules
+from sec_certs import helpers, dataset, extract_certificates, constants as constants
+from sec_certs.cert_rules import fips_common_rules, REGEXEC_SEP, fips_rules
+from sec_certs.certificate.certificate import Certificate, logger
 from sec_certs.configuration import config
-from sec_certs.cpe import CPE, CPEDataset
-from sec_certs.cve import CVE, CVEDataset
-
-logger = logging.getLogger(__name__)
-
-
-class Certificate(ABC):
-    T = TypeVar('T', bound='Certificate')
-
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def __repr__(self) -> str:
-        return str(self.to_dict())
-
-    def __str__(self) -> str:
-        return 'Not implemented'
-
-    @property
-    @abstractmethod
-    def dgst(self):
-        raise NotImplementedError('Not meant to be implemented')
-
-    def __eq__(self, other: 'Certificate') -> bool:
-        return self.dgst == other.dgst
-
-    def to_dict(self):
-        return {**{'dgst': self.dgst}, **copy.deepcopy(self.__dict__)}
-
-    @classmethod
-    def from_dict(cls: Type[T], dct: dict) -> T:
-        dct.pop('dgst')
-        return cls(*(tuple(dct.values())))
-
-    def to_json(self, output_path: Union[Path, str]):
-        with Path(output_path).open('w') as handle:
-            json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
-
-    @classmethod
-    def from_json(cls, input_path: Union[Path, str]):
-        with Path(input_path).open('r') as handle:
-            return json.load(handle, cls=CustomJSONDecoder)
+from sec_certs.extract_certificates import load_cert_file, LINE_SEPARATOR, save_modified_cert_file, \
+    normalize_match_string
+from sec_certs.serialization import ComplexSerializableType
 
 
 class FIPSCertificate(Certificate, ComplexSerializableType):
@@ -743,539 +695,4 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
     def get_compare(vendor: str):
         vendor_split = vendor.replace(',', '') \
             .replace('-', ' ').replace('+', ' ').replace('®', '').split()
-        return vendor_split[0] if len(vendor_split) > 0 else vendor
-
-
-class CommonCriteriaCert(Certificate, ComplexSerializableType):
-    cc_url = 'http://www.commoncriteriaportal.org'
-    empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/'
-
-    @dataclass(eq=True, frozen=True)
-    class MaintainanceReport(ComplexSerializableType):
-        """
-        Object for holding maintainance reports.
-        """
-        maintainance_date: date
-        maintainance_title: str
-        maintainance_report_link: str
-        maintainance_st_link: str
-
-        def __post_init__(self):
-            super().__setattr__('maintainance_report_link',
-                                helpers.sanitize_link(self.maintainance_report_link))
-            super().__setattr__('maintainance_st_link',
-                                helpers.sanitize_link(self.maintainance_st_link))
-            super().__setattr__('maintainance_title',
-                                helpers.sanitize_string(self.maintainance_title))
-            super().__setattr__('maintainance_date', helpers.sanitize_date(self.maintainance_date))
-
-        def to_dict(self):
-            return copy.deepcopy(self.__dict__)
-
-        @classmethod
-        def from_dict(cls, dct):
-            return cls(*tuple(dct.values()))
-
-        def __lt__(self, other):
-            return self.maintainance_date < other.maintainance_date
-
-    @dataclass(eq=True, frozen=True)
-    class ProtectionProfile(ComplexSerializableType):
-        """
-        Object for holding protection profiles.
-        """
-        pp_name: str
-        pp_link: Optional[str]
-
-        def __post_init__(self):
-            super().__setattr__('pp_name', helpers.sanitize_string(self.pp_name))
-            super().__setattr__('pp_link', helpers.sanitize_link(self.pp_link))
-
-        def to_dict(self):
-            return copy.deepcopy(self.__dict__)
-
-        @classmethod
-        def from_dict(cls, dct):
-            return cls(*tuple(dct.values()))
-
-        def __lt__(self, other):
-            return self.pp_name < other.pp_name
-
-    @dataclass(init=False)
-    class InternalState(ComplexSerializableType):
-        st_link_ok: bool
-        report_link_ok: bool
-        st_convert_ok: bool
-        report_convert_ok: bool
-        st_extract_ok: bool
-        report_extract_ok: bool
-        st_pdf_path: Path
-        report_pdf_path: Path
-        st_txt_path: Path
-        report_txt_path: Path
-        errors: Optional[List[str]]
-
-        def __init__(self, st_link_ok: bool = True, report_link_ok: bool = True,
-                     st_convert_ok: bool = True, report_convert_ok: bool = True,
-                     st_extract_ok: bool = True, report_extract_ok: bool = True,
-                     errors: Optional[List[str]] = None):
-            self.st_link_ok = st_link_ok
-            self.report_link_ok = report_link_ok
-            self.st_convert_ok = st_convert_ok
-            self.report_convert_ok = report_convert_ok
-            self.st_extract_ok = st_extract_ok
-            self.report_extract_ok = report_extract_ok
-
-            if errors is None:
-                self.errors = []
-            else:
-                self.errors = errors
-
-        def to_dict(self):
-            return {'st_link_ok': self.st_link_ok, 'report_link_ok': self.report_link_ok,
-                    'st_convert_ok': self.st_convert_ok, 'report_convert_ok': self.report_convert_ok,
-                    'st_extract_ok': self.st_extract_ok, 'report_extract_ok': self.report_extract_ok,
-                    'errors': self.errors}
-
-        @classmethod
-        def from_dict(cls, dct: Dict[str, bool]):
-            return cls(*tuple(dct.values()))
-
-    @dataclass(init=False)
-    class PdfData(ComplexSerializableType):
-        report_metadata: Dict[str, str]
-        st_metadata: Dict[str, str]
-        report_frontpage: Dict[str, str]
-        st_frontpage: Dict[str, str]
-        report_keywords: Dict[str, str]
-        st_keywords: Dict[str, str]
-
-        def __init__(self, report_metadata: Optional[Dict[str, str]] = None,
-                     st_metadata: Optional[Dict[str, str]] = None,
-                     report_frontpage: Optional[Dict[str, str]] = None, st_frontpage: Optional[Dict[str, str]] = None,
-                     report_keywords: Optional[Dict[str, str]] = None, st_keywords: Optional[Dict[str, str]] = None):
-            self.report_metadata = report_metadata
-            self.st_metadata = st_metadata
-            self.report_frontpage = report_frontpage
-            self.st_frontpage = st_frontpage
-            self.report_keywords = report_keywords
-            self.st_keywords = st_keywords
-
-        def to_dict(self):
-            return {'report_metadata': self.report_metadata, 'st_metadata': self.st_metadata,
-                    'report_frontpage': self.report_frontpage,
-                    'st_frontpage': self.st_frontpage, 'report_keywords': self.report_keywords,
-                    'st_keywords': self.st_keywords}
-
-        @classmethod
-        def from_dict(cls, dct: Dict[str, bool]):
-            return cls(*tuple(dct.values()))
-
-    @dataclass(init=False)
-    class Heuristics(ComplexSerializableType):
-        extracted_versions: List[str]
-        cpe_candidate_vendors: Optional[List[str]] = field(init=False)
-        cpe_matches: Optional[List[Tuple[float, CPE]]]
-        verified_cpe_matches: Optional[List[CPE]]
-        related_cves: Optional[List[str]]
-
-        def __init__(self,
-                     extracted_versions: Optional[List[str]] = None,
-                     cpe_matches: Optional[List[str]] = None,
-                     verified_cpe_matches: Optional[List[str]] = None,
-                     related_cves: Optional[List[CVE]] = None):
-            self.extracted_versions = extracted_versions
-            self.cpe_matches = cpe_matches
-            self.cpe_candidate_vendors = None
-            self.verified_cpe_matches = verified_cpe_matches
-            self.related_cves = related_cves
-
-        def to_dict(self):
-            return {'extracted_versions': self.extracted_versions, 'cpe_matches': self.cpe_matches, 'verified_cpe_matches': self.verified_cpe_matches, 'related_cves': self.related_cves}
-
-        @classmethod
-        def from_dict(cls, dct: Dict[str, str]):
-            return cls(*tuple(dct.values()))
-
-    pandas_columns = ['dgst', 'name', 'status', 'category', 'manufacturer', 'scheme', 'security_level',
-                      'not_valid_before', 'not_valid_after', 'report_link', 'st_link',
-                      'manufacturer_web', 'extracted_versions', 'cpe_matches', 'verified_cpe_matches',
-                      'related_cves']
-
-    def __init__(self, status: str, category: str, name: str, manufacturer: str, scheme: str,
-                 security_level: Union[str, set], not_valid_before: date,
-                 not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str],
-                 manufacturer_web: Optional[str],
-                 protection_profiles: set,
-                 maintainance_updates: set,
-                 state: Optional[InternalState],
-                 pdf_data: Optional[PdfData],
-                 heuristics: Optional[Heuristics]):
-        super().__init__()
-
-        self.status = status
-        self.category = category
-        self.name = helpers.sanitize_string(name)
-        self.manufacturer = helpers.sanitize_string(manufacturer)
-        self.scheme = scheme
-        self.security_level = helpers.sanitize_security_levels(security_level)
-        self.not_valid_before = helpers.sanitize_date(not_valid_before)
-        self.not_valid_after = helpers.sanitize_date(not_valid_after)
-        self.report_link = helpers.sanitize_link(report_link)
-        self.st_link = helpers.sanitize_link(st_link)
-        self.src = src
-        self.cert_link = helpers.sanitize_link(cert_link)
-        self.manufacturer_web = helpers.sanitize_link(manufacturer_web)
-        self.protection_profiles = protection_profiles
-        self.maintainance_updates = maintainance_updates
-
-        if state is None:
-            state = self.InternalState()
-        self.state = state
-
-        if pdf_data is None:
-            pdf_data = self.PdfData()
-        self.pdf_data = pdf_data
-
-        if heuristics is None:
-            heuristics = self.Heuristics()
-        self.heuristics = heuristics
-
-    @property
-    def dgst(self) -> str:
-        """
-        Computes the primary key of the certificate using first 16 bytes of SHA-256 digest
-        """
-        return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link)
-
-    def __str__(self):
-        return self.manufacturer + ' ' + self.name + ' dgst: ' + self.dgst
-
-    def to_pandas_tuple(self):
-        return self.dgst, self.name, self.status, self.category, self.manufacturer, self.scheme, self.security_level,\
-               self.not_valid_before, self.not_valid_after, self.report_link, self.st_link, self.manufacturer_web, \
-               self.heuristics.extracted_versions, self.heuristics.cpe_matches, self.heuristics.verified_cpe_matches, \
-               self.heuristics.related_cves
-
-
-    def merge(self, other: 'CommonCriteriaCert'):
-        """
-        Merges with other CC certificate. Assuming they come from different sources, e.g., csv and html.
-        Assuming that html source has better protection profiles, they overwrite CSV info
-        On other values (apart from maintainances, see TODO below) the sanity checks are made.
-        """
-        if self != other:
-            logger.warning(
-                f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}')
-
-        for att, val in vars(self).items():
-            if not val:
-                setattr(self, att, getattr(other, att))
-            elif self.src == 'csv' and other.src == 'html' and att == 'protection_profiles':
-                setattr(self, att, getattr(other, att))
-            elif self.src == 'csv' and other.src == 'html' and att == 'maintainance_updates':
-                # TODO Fix me: This is a simplification. At the moment html contains more reliable info
-                setattr(self, att, getattr(other, att))
-            elif att == 'src':
-                pass  # This is expected
-            elif att == 'state':
-                setattr(self, att, getattr(other, att))
-            else:
-                if getattr(self, att) != getattr(other, att):
-                    logger.warning(
-                        f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}')
-        if self.src != other.src:
-            self.src = self.src + ' + ' + other.src
-
-    @classmethod
-    def from_dict(cls, dct: Dict) -> 'CommonCriteriaCert':
-        new_dct = dct.copy()
-        new_dct['maintainance_updates'] = set(dct['maintainance_updates'])
-        new_dct['protection_profiles'] = set(dct['protection_profiles'])
-        return super(cls, CommonCriteriaCert).from_dict(new_dct)
-
-    @classmethod
-    def from_html_row(cls, row: Tag, status: str, category: str) -> 'CommonCriteriaCert':
-        """
-        Creates a CC certificate from html row
-        """
-
-        def _get_name(cell: Tag) -> str:
-            return list(cell.stripped_strings)[0]
-
-        def _get_manufacturer(cell: Tag) -> Optional[str]:
-            if lst := list(cell.stripped_strings):
-                return lst[0]
-            else:
-                return None
-
-        def _get_scheme(cell: Tag) -> str:
-            return list(cell.stripped_strings)[0]
-
-        def _get_security_level(cell: Tag) -> set:
-            return set(cell.stripped_strings)
-
-        def _get_manufacturer_web(cell: Tag) -> Optional[str]:
-            for link in cell.find_all('a'):
-                if link is not None and link.get('title') == 'Vendor\'s web site' and link.get('href') != 'http://':
-                    return link.get('href')
-            return None
-
-        def _get_protection_profiles(cell: Tag) -> set:
-            protection_profiles = set()
-            for link in list(cell.find_all('a')):
-                if link.get('href') is not None and '/ppfiles/' in link.get('href'):
-                    protection_profiles.add(CommonCriteriaCert.ProtectionProfile(str(link.contents[0]),
-                                                                                 CommonCriteriaCert.cc_url + link.get(
-                                                                                     'href')))
-            return protection_profiles
-
-        def _get_date(cell: Tag) -> date:
-            text = cell.get_text()
-            extracted_date = datetime.strptime(
-                text, '%Y-%m-%d').date() if text else None
-            return extracted_date
-
-        def _get_report_st_links(cell: Tag) -> (str, str):
-            links = cell.find_all('a')
-            # TODO: Exception checks
-            assert links[1].get('title').startswith('Certification Report')
-            assert links[2].get('title').startswith('Security Target')
-
-            report_link = CommonCriteriaCert.cc_url + links[1].get('href')
-            security_target_link = CommonCriteriaCert.cc_url + \
-                                   links[2].get('href')
-
-            return report_link, security_target_link
-
-        def _get_cert_link(cell: Tag) -> Optional[str]:
-            links = cell.find_all('a')
-            return CommonCriteriaCert.cc_url + links[0].get('href') if links else None
-
-        def _get_maintainance_div(cell: Tag) -> Optional[Tag]:
-            divs = cell.find_all('div')
-            for d in divs:
-                if d.find('div') and d.stripped_strings and list(d.stripped_strings)[0] == 'Maintenance Report(s)':
-                    return d
-            return None
-
-        def _get_maintainance_updates(main_div: Tag) -> set:
-            possible_updates = list(main_div.find_all('li'))
-            maintainance_updates = set()
-            for u in possible_updates:
-                text = list(u.stripped_strings)[0]
-                main_date = datetime.strptime(text.split(
-                    ' ')[0], '%Y-%m-%d').date() if text else None
-                main_title = text.split('– ')[1]
-                main_report_link = None
-                main_st_link = None
-                links = u.find_all('a')
-                for l in links:
-                    if l.get('title').startswith('Maintenance Report:'):
-                        main_report_link = CommonCriteriaCert.cc_url + \
-                                           l.get('href')
-                    elif l.get('title').startswith('Maintenance ST'):
-                        main_st_link = CommonCriteriaCert.cc_url + \
-                                       l.get('href')
-                    else:
-                        logger.error('Unknown link in Maintenance part!')
-                maintainance_updates.add(
-                    CommonCriteriaCert.MaintainanceReport(main_date, main_title, main_report_link, main_st_link))
-            return maintainance_updates
-
-        cells = list(row.find_all('td'))
-        if len(cells) != 7:
-            logger.error('Unexpected number of cells in CC html row.')
-            raise
-
-        name = _get_name(cells[0])
-        manufacturer = _get_manufacturer(cells[1])
-        manufacturer_web = _get_manufacturer_web(cells[1])
-        scheme = _get_scheme(cells[6])
-        security_level = _get_security_level(cells[5])
-        protection_profiles = _get_protection_profiles(cells[0])
-        not_valid_before = _get_date(cells[3])
-        not_valid_after = _get_date(cells[4])
-        report_link, st_link = _get_report_st_links(cells[0])
-        cert_link = _get_cert_link(cells[2])
-
-        maintainance_div = _get_maintainance_div(cells[0])
-        maintainances = _get_maintainance_updates(
-            maintainance_div) if maintainance_div else set()
-
-        return cls(status, category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after,
-                   report_link,
-                   st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None, None, None)
-
-    def set_local_paths(self,
-                        report_pdf_dir: Optional[Union[str, Path]],
-                        st_pdf_dir: Optional[Union[str, Path]],
-                        report_txt_dir: Optional[Union[str, Path]],
-                        st_txt_dir: Optional[Union[str, Path]]):
-        if report_pdf_dir is not None:
-            self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + '.pdf')
-        if st_pdf_dir is not None:
-            self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + '.pdf')
-        if report_txt_dir is not None:
-            self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + '.txt')
-        if st_txt_dir is not None:
-            self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + '.txt')
-
-    @property
-    def best_cpe_match(self):
-        clean = [x for x in self.cpe_matching if len(x[0]) > 5]
-        cpe_match_ranking = [x[1] for x in clean]
-        argmax = cpe_match_ranking.index(max(cpe_match_ranking))
-        return clean[argmax]
-
-    @staticmethod
-    def download_pdf_report(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path)
-        if exit_code != requests.codes.ok:
-            error_msg = f'failed to download report from {cert.report_link}, code: {exit_code}'
-            logger.error(f'Cert dgst: {cert.dgst} ' + error_msg)
-            cert.state.report_link_ok = False
-            cert.state.errors.append(error_msg)
-        return cert
-
-    @staticmethod
-    def download_pdf_target(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path)
-        if exit_code != requests.codes.ok:
-            error_msg = f'failed to download ST from {cert.report_link}, code: {exit_code}'
-            logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
-            cert.state.st_link_ok = False
-            cert.state.errors.append(error_msg)
-        return cert
-
-    def path_is_corrupted(self, local_path):
-        return not local_path.exists() or local_path.stat().st_size < constants.MIN_CORRECT_CERT_SIZE
-
-    @staticmethod
-    def convert_report_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path, ['-raw'])
-        if exit_code != constants.RETURNCODE_OK:
-            error_msg = 'failed to convert report pdf->txt'
-            logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
-            cert.state.report_convert_ok = False
-            cert.state.errors.append(error_msg)
-        return cert
-
-    @staticmethod
-    def convert_target_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path, ['-raw'])
-        if exit_code != constants.RETURNCODE_OK:
-            error_msg = 'failed to convert security target pdf->txt'
-            logger.error(f'Cert dgst: {cert.dgst}' + error_msg)
-            cert.state.st_convert_ok = False
-            cert.state.errors.append(error_msg)
-        return cert
-
-    @staticmethod
-    def extract_st_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        response, cert.pdf_data.st_metadata = helpers.extract_pdf_metadata(cert.state.st_pdf_path)
-        if response != constants.RETURNCODE_OK:
-            cert.state.st_extract_ok = False
-            cert.state.errors.append(response)
-        return cert
-
-    @staticmethod
-    def extract_report_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        response, cert.pdf_data.report_metadata = helpers.extract_pdf_metadata(cert.state.report_pdf_path)
-        if response != constants.RETURNCODE_OK:
-            cert.state.report_extract_ok = False
-            cert.state.errors.append(response)
-        return cert
-
-    @staticmethod
-    def extract_st_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        cert.pdf_data.st_frontpage = dict()
-
-        response_anssi, cert.pdf_data.st_frontpage['anssi'] = helpers.search_only_headers_anssi(cert.state.st_txt_path)
-        response_bsi, cert.pdf_data.st_frontpage['bsi'] = helpers.search_only_headers_bsi(cert.state.st_txt_path)
-
-        if response_anssi != constants.RETURNCODE_OK:
-            cert.state.st_extract_ok = False
-            cert.state.errors.append(response_anssi)
-        if response_bsi != constants.RETURNCODE_OK:
-            cert.state.st_extract_ok = False
-            cert.state.errors.append(response_bsi)
-
-        return cert
-
-    @staticmethod
-    def extract_report_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        cert.pdf_data.report_frontpage = dict()
-        response_bsi, cert.pdf_data.report_frontpage['bsi'] = helpers.search_only_headers_bsi(
-            cert.state.report_txt_path)
-        response_anssi, cert.pdf_data.report_frontpage['anssi'] = helpers.search_only_headers_anssi(
-            cert.state.report_txt_path)
-
-        if response_anssi != constants.RETURNCODE_OK:
-            cert.state.report_extract_ok = False
-            cert.state.errors.append(response_anssi)
-        if response_bsi != constants.RETURNCODE_OK:
-            cert.state.report_extract_ok = False
-            cert.state.errors.append(response_bsi)
-
-        return cert
-
-    @staticmethod
-    def extract_report_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        response, cert.pdf_data.report_keywords = helpers.extract_keywords(cert.state.report_txt_path)
-        if response != constants.RETURNCODE_OK:
-            cert.state.report_extract_ok = False
-        return cert
-
-    @staticmethod
-    def extract_st_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
-        response, cert.pdf_data.st_keywords = helpers.extract_keywords(cert.state.st_txt_path)
-        if response != constants.RETURNCODE_OK:
-            cert.state.st_extract_ok = False
-            cert.state.errors.append(response)
-        return cert
-
-    def compute_heuristics_version(self):
-        """
-        Will extract possible versions from the name
-        """
-        at_least_something = r'(\b(\d)+\b)'
-        just_numbers = r'(\d{1,5})(\.\d{1,5})'
-
-        without_version = r'(' + just_numbers + r'+)'
-        long_version = r'(' + r'(\bversion)\s*' + just_numbers + r'+)'
-        short_version = r'(' + r'\bv\s*' + just_numbers + r'+)'
-        full_regex_string = r'|'.join([without_version, short_version, long_version])
-        normalizer = r'(\d+\.*)+'
-
-        matched_strings = set([max(x, key=len) for x in re.findall(full_regex_string, self.name, re.IGNORECASE)])
-        if not matched_strings:
-            matched_strings = set([max(x, key=len) for x in re.findall(at_least_something, self.name, re.IGNORECASE)])
-
-        if matched_strings:
-            self.heuristics.extracted_versions = [re.search(normalizer, x).group() for x in matched_strings]
-        else:
-            self.heuristics.extracted_versions = ['-']
-
-    def compute_heuristics_cpe_vendors(self, cpe_dataset: CPEDataset):
-        """
-        With the help of the CPE dataset, will find CPE vendors that could match the given certificate vendor
-        """
-        self.heuristics.cpe_candidate_vendors = cpe_dataset.get_candidate_list_of_vendors(self.manufacturer)
-
-    def compute_heuristics_cpe_match(self, cpe_dataset: CPEDataset):
-        self.compute_heuristics_cpe_vendors(cpe_dataset)
-        self.heuristics.cpe_matches = cpe_dataset.get_cpe_matches(self.name,
-                                                                  self.heuristics.cpe_candidate_vendors,
-                                                                  self.heuristics.extracted_versions,
-                                                                  n_max_matches=constants.CPE_MAX_MATCHES,
-                                                                  threshold=constants.CPE_MATCHING_THRESHOLD)
-
-    def compute_heuristics_related_cves(self, cve_dataset: CVEDataset):
-        if self.heuristics.verified_cpe_matches:
-            related_cves = [cve_dataset.get_cves_for_cpe(x.uri) for x in self.heuristics.verified_cpe_matches]
-            related_cves = list(filter(lambda x: x is not None, related_cves))
-            if related_cves:
-                self.heuristics.related_cves = list(itertools.chain.from_iterable(related_cves))
-        else:
-            self.heuristics.related_cves = None
\ No newline at end of file
+        return vendor_split[0] if len(vendor_split) > 0 else vendor
\ No newline at end of file
diff --git a/sec_certs/dataset.py b/sec_certs/dataset/common_criteria.py
index 5845e4d3..18326b10 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -1,140 +1,21 @@
-import os
-from datetime import datetime
-import locale
-import logging
-from typing import Dict, List, ClassVar, Collection, Union, Set, Tuple, Optional
-from itertools import groupby
-from dataclasses import dataclass
 import copy
+import locale
+import shutil
 import time
-
-import json
-from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import datetime
 from pathlib import Path
-import shutil
+from typing import Dict, Optional, Union, List
 
-from graphviz import Digraph
-import requests
 import pandas as pd
-from bs4 import BeautifulSoup, Tag
-from rapidfuzz import process, fuzz
-import xml.etree.ElementTree as ET
-
-import sec_certs.helpers as helpers
-import sec_certs.constants as constants
-import sec_certs.cert_processing as cert_processing
-import sec_certs.files as files
-
-from sec_certs.certificate import CommonCriteriaCert, Certificate, FIPSCertificate
-from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
-from sec_certs.configuration import config
-from sec_certs.cpe import CPEDataset
-from sec_certs.cve import CVEDataset
-
-logger = logging.getLogger(__name__)
-
-
-class Dataset(ABC):
-    def __init__(self, certs: Dict[str, 'Certificate'], root_dir: Path, name: str = 'dataset name',
-                 description: str = 'dataset_description'):
-        self._root_dir = root_dir
-        self.timestamp = datetime.now()
-        self.sha256_digest = 'not implemented'
-        self.name = name
-        self.description = description
-        self.certs = certs
-
-    @property
-    def root_dir(self):
-        return self._root_dir
-
-    @root_dir.setter
-    def root_dir(self, new_dir: Union[str, Path]):
-        if not (new_path := Path(new_dir)).exists():
-            raise FileNotFoundError('Root directory for Dataset does not exist')
-        self._root_dir = new_path
-
-    @property
-    def json_path(self) -> Path:
-        return self.root_dir / (self.name + '.json')
-
-    def __iter__(self):
-        yield from self.certs.values()
-
-    def __getitem__(self, item: str):
-        return self.certs.__getitem__(item.lower())
-
-    def __setitem__(self, key: str, value: 'Certificate'):
-        self.certs.__setitem__(key.lower(), value)
-
-    def __len__(self) -> int:
-        return len(self.certs)
-
-    def __eq__(self, other: 'Dataset') -> bool:
-        return self.certs == other.certs
-
-    def __str__(self) -> str:
-        return str(type(self).__name__) + ':' + self.name + ', ' + str(len(self)) + ' certificates'
-
-    def to_dict(self):
-        return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest,
-                'name': self.name, 'description': self.description,
-                'n_certs': len(self), 'certs': list(self.certs.values())}
-
-    @classmethod
-    def from_dict(cls, dct: Dict):
-        certs = {x.dgst: x for x in dct['certs']}
-        dset = cls(certs, Path('./'), dct['name'], dct['description'])
-        if len(dset) != (claimed := dct['n_certs']):
-            logger.error(
-                f'The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed}).')
-        return dset
-
-    def to_json(self, output_path: Union[str, Path] = None):
-        if not output_path:
-            output_path = self.json_path
-
-        with Path(output_path).open('w') as handle:
-            json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
-
-    @classmethod
-    def from_json(cls, input_path: Union[str, Path]):
-        input_path = Path(input_path)
-        with input_path.open('r') as handle:
-            dset = json.load(handle, cls=CustomJSONDecoder)
-        dset.root_dir = input_path.parent.absolute()
-        return dset
-
-    @abstractmethod
-    def get_certs_from_web(self):
-        raise NotImplementedError('Not meant to be implemented by the base class.')
-
-    @abstractmethod
-    def convert_all_pdfs(self):
-        raise NotImplementedError('Not meant to be implemented by the base class.')
+from bs4 import Tag, BeautifulSoup
 
-    @abstractmethod
-    def download_all_pdfs(self):
-        raise NotImplementedError('Not meant to be implemented by the base class.')
-
-    @staticmethod
-    def _download_parallel(urls: Collection[str], paths: Collection[Path], prune_corrupted: bool = True):
-        exit_codes = cert_processing.process_parallel(helpers.download_file,
-                                                      list(zip(urls, paths)),
-                                                      constants.N_THREADS,
-                                                      unpack=True)
-        n_successful = len([e for e in exit_codes if e == requests.codes.ok])
-        logger.info(f'Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.')
-
-        for url, e in zip(urls, exit_codes):
-            if e != requests.codes.ok:
-                logger.error(f'Failed to download {url}, exit code: {e}')
-
-        if prune_corrupted is True:
-            for p in paths:
-                if p.exists() and p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE:
-                    logger.error(f'Corrupted file at: {p}')
-                    # TODO: Delete
+from sec_certs import helpers as helpers, cert_processing as cert_processing, constants as constants
+from sec_certs.dataset.cpe import CPEDataset
+from sec_certs.dataset.cve import CVEDataset
+from sec_certs.dataset.dataset import Dataset, logger
+from sec_certs.serialization import ComplexSerializableType
+from sec_certs.certificate.common_criteria import CommonCriteriaCert
 
 
 class CCDataset(Dataset, ComplexSerializableType):
@@ -774,477 +655,4 @@ class CCDataset(Dataset, ComplexSerializableType):
             logger.error('No certificates with verified CPE match detected. You must run dset.manually_verify_cpe_matches() first. Returning.')
             return
         for cert in verified_cpe_rich_certs:
-            cert.compute_heuristics_related_cves(cve_dset)
-
-
-class FIPSDataset(Dataset, ComplexSerializableType):
-    FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov'
-    FIPS_MODULE_URL: ClassVar[
-        str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
-
-    def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name',
-                 description: str = 'dataset_description'):
-        super().__init__(certs, root_dir, name, description)
-        self.keywords = {}
-        self.algorithms = None
-        self.new_files = 0
-
-    @property
-    def web_dir(self) -> Path:
-        return self.root_dir / 'web'
-
-    @property
-    def results_dir(self) -> Path:
-        return self.root_dir / 'results'
-
-    @property
-    def policies_dir(self) -> Path:
-        return self.root_dir / 'security_policies'
-
-    @property
-    def fragments_dir(self) -> Path:
-        return self.root_dir / 'fragments'
-
-    @property
-    def algs_dir(self) -> Path:
-        return self.web_dir / 'algorithms'
-
-    def find_empty_pdfs(self) -> Tuple[List, List]:
-        missing = []
-        not_available = []
-        for i in self.certs:
-            if not (self.policies_dir / f'{i}.pdf').exists():
-                missing.append(i)
-            elif os.path.getsize(self.policies_dir / f'{i}.pdf') < constants.FIPS_NOT_AVAILABLE_CERT_SIZE:
-                not_available.append(i)
-        return missing, not_available
-
-    def extract_keywords(self, redo=False):
-        self.fragments_dir.mkdir(parents=True, exist_ok=True)
-
-        keywords = cert_processing.process_parallel(FIPSCertificate.find_keywords,
-                                                    [cert for cert in self.certs.values() if
-                                                     not cert.pdf_scan.keywords or redo],
-                                                    constants.N_THREADS,
-                                                    use_threading=False)
-        for keyword, cert in keywords:
-            self.certs[cert.dgst].pdf_scan.keywords = keyword
-
-    def match_algs(self, show_graph=False) -> Dict:
-        output = {}
-        for cert in self.certs.values():
-            output[cert.dgst] = FIPSCertificate.match_web_algs_to_pdf(cert)
-
-        return output
-
-
-    def download_all_pdfs(self):
-        sp_paths, sp_urls = [], []
-        self.policies_dir.mkdir(exist_ok=True)
-
-        for cert_id in list(self.certs.keys()):
-            if not (self.policies_dir / f'{cert_id}.pdf').exists() or not self.certs[cert_id].state.txt_state:
-                sp_urls.append(
-                    f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf")
-                sp_paths.append(self.policies_dir / f"{cert_id}.pdf")
-        logging.info(f"downloading {len(sp_urls)} module pdf files")
-        cert_processing.process_parallel(FIPSCertificate.download_security_policy, list(zip(sp_urls, sp_paths)),
-                                         constants.N_THREADS)
-        self.new_files += len(sp_urls)
-
-    def download_all_htmls(self) -> List[str]:
-        html_paths, html_urls = [], []
-        new_files = []
-        self.web_dir.mkdir(exist_ok=True)
-        for cert_id in self.certs.keys():
-            if not (self.web_dir / f'{cert_id}.html').exists():
-                html_urls.append(
-                    f"https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{cert_id}")
-                html_paths.append(self.web_dir / f"{cert_id}.html")
-                new_files.append(cert_id)
-
-        logging.info(f"downloading {len(html_urls)} module html files")
-        failed = cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(html_urls, html_paths)),
-                                         constants.N_THREADS)
-        failed = [c for c in failed if c]
-
-        self.new_files += len(html_urls)
-        logging.info(f"Download failed for {len(failed)} files. Retrying...")
-        cert_processing.process_parallel(FIPSCertificate.download_html_page, failed,
-                                         constants.N_THREADS)
-        return new_files
-
-    def convert_all_pdfs(self):
-        logger.info('Converting FIPS certificate reports to .txt')
-        tuples = [
-            (cert, self.policies_dir / f'{cert.cert_id}.pdf', self.policies_dir / f'{cert.cert_id}.pdf.txt')
-            for cert in self.certs.values() if
-            not cert.state.txt_state and (self.policies_dir / f'{cert.cert_id}.pdf').exists()
-        ]
-        cert_processing.process_parallel(FIPSCertificate.convert_pdf_file, tuples, constants.N_THREADS)
-
-    def get_certs_from_web(self, redo: bool = False, json_file: Optional[Path] = None):
-        def download_html_pages() -> List[str]:
-            new_files = self.download_all_htmls()
-            self.download_all_pdfs()
-            return new_files
-
-        def get_certificates_from_html(html_file: Path) -> None:
-            logger.info(f'Getting certificate ids from {html_file}')
-            with open(html_file, 'r', encoding='utf-8') as handle:
-                html = BeautifulSoup(handle.read(), 'html.parser')
-
-            table = [x for x in html.find(
-                id='searchResultsTable').tbody.contents if x != '\n']
-            for entry in table:
-                self.certs[entry.find('a').text] = {}
-
-        logger.info("Downloading required html files")
-
-        self.web_dir.mkdir(parents=True, exist_ok=True)
-        self.policies_dir.mkdir(exist_ok=True)
-        self.algs_dir.mkdir(exist_ok=True)
-
-        # Download files containing all available module certs (always)
-        html_files = ['fips_modules_active.html',
-                      'fips_modules_historical.html', 'fips_modules_revoked.html']
-        helpers.download_file(
-            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0",
-            self.web_dir / "fips_modules_active.html")
-        helpers.download_file(
-            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Historical&ValidationYear=0",
-            self.web_dir / "fips_modules_historical.html")
-        helpers.download_file(
-            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Revoked&ValidationYear=0",
-            self.web_dir / "fips_modules_revoked.html")
-
-        # Parse those files and get list of currently processable files (always)
-        for f in html_files:
-            get_certificates_from_html(self.web_dir / f)
-
-        logger.info('Downloading certificate html and security policies')
-
-        if not json_file:
-            json_file = self.root_dir / 'fips_full_dataset.json'
-
-        if json_file.exists():
-            logger.info("Certs loaded from previous scanning")
-            dataset = self.from_json(json_file)
-            self.certs = dataset.certs
-            self.algorithms = dataset.algorithms
-
-        new_certs = download_html_pages()
-
-        logger.info(f"{self.new_files} needed to be downloaded")
-
-        for cert_id in new_certs:
-            self.certs[cert_id] = None
-
-        if not redo and self.new_files == 0:
-            logger.info('No new changes to web_scan are going to be made')
-            return
-        # now we want to do redo, because we want to avoid duplicites
-        redo = True
-        logger.info(f'Parsing web pages{" from scratch" if redo else ""}...')
-        for cert_id, cert in self.certs.items():
-            self.certs[cert_id] = FIPSCertificate.html_from_file(
-                self.web_dir / f'{cert_id}.html',
-                FIPSCertificate.State((self.policies_dir / cert_id).with_suffix('.pdf'),
-                                      (self.web_dir / cert_id).with_suffix('.html'),
-                                      (self.fragments_dir / cert_id).with_suffix('.txt'), False, None, False),
-                cert, redo=redo)
-
-    def extract_certs_from_tables(self) -> List[Path]:
-        """
-        Function that extracts algorithm IDs from tables in security policies files.
-        :return: list of files that couldn't have been decoded
-        """
-        result = cert_processing.process_parallel(FIPSCertificate.analyze_tables,
-                                                  [cert for cert in self.certs.values() if
-                                                   not cert.state.tables_done and cert.state.txt_state],
-                                                  constants.N_THREADS // 4,  # tabula already processes by parallel, so
-                                                  # it's counterproductive to use all threads
-                                                  use_threading=False)
-
-        not_decoded = [cert.state.sp_path for done, cert, _ in result if done is False]
-        for state, cert, algorithms in result:
-            self.certs[cert.dgst].state.tables_done = state
-            self.certs[cert.dgst].pdf_scan.algorithms += algorithms
-
-        return not_decoded
-
-    def remove_algorithms_from_extracted_data(self):
-        for cert in self.certs.values():
-            cert.remove_algorithms()
-
-    def unify_algorithms(self):
-        certificate: FIPSCertificate
-        for certificate in self.certs.values():
-            new_algorithms = []
-            united_algorithms = [x for x in (certificate.web_scan.algorithms + certificate.pdf_scan.algorithms) if
-                                 x != {'Certificate': []}]
-            for algorithm in united_algorithms:
-                if isinstance(algorithm, dict):
-                    new_algorithms.append(algorithm)
-                else:
-                    new_algorithms.append({'Certificate': [algorithm]})
-            certificate.processed.algorithms = new_algorithms
-
-    def validate_results(self):
-        """
-        Function that validates results and finds the final connection output
-        """
-
-        def validate_id(processed_cert: FIPSCertificate, cert_candidate: str) -> bool:
-
-            # returns True if candidates should _not_ be matched
-            def compare_certs(current_certificate: 'FIPSCertificate', other_id: str):
-                cert_first = current_certificate.web_scan.date_validation[0].year
-                cert_last = current_certificate.web_scan.date_validation[-1].year
-                conn_first = self.certs[other_id].web_scan.date_validation[0].year
-                conn_last = self.certs[other_id].web_scan.date_validation[-1].year
-
-                return cert_first - conn_first > config.year_difference_between_validations['value'] \
-                       and cert_last - conn_last > config.year_difference_between_validations['value'] \
-                       or cert_first < conn_first
-
-            # "< number" still needs to be used, because of some old certs being revalidated
-            if cert_candidate.isdecimal() \
-                    and int(cert_candidate) < config.smallest_certificate_id_to_connect['value'] or \
-                    compare_certs(processed_cert, cert_candidate):
-                return False
-            if cert_candidate not in self.algorithms.certs:
-                return True
-
-            for cert_alg in processed_cert.processed.algorithms:
-                for certificate in cert_alg['Certificate']:
-                    curr_id = ''.join(filter(str.isdigit, certificate))
-                    if curr_id == cert_candidate:
-                        return False
-
-            algs = self.algorithms.certs[cert_candidate]
-            for current_alg in algs:
-                if FIPSCertificate.get_compare(processed_cert.web_scan.vendor) == FIPSCertificate.get_compare(
-                        current_alg.vendor):
-                    return False
-            return True
-
-        broken_files = set()
-
-        current_cert: FIPSCertificate
-
-        for current_cert in self.certs.values():
-            if not current_cert.state.txt_state:
-                continue
-            for rule in current_cert.processed.keywords['rules_cert_id']:
-                for cert in current_cert.processed.keywords['rules_cert_id'][rule]:
-                    cert_id = ''.join(filter(str.isdigit, cert))
-
-                    if cert_id == '' or cert_id not in self.certs:
-                        broken_files.add(current_cert.dgst)
-                        current_cert.state.file_status = False
-                        break
-
-        if broken_files:
-            logger.warning("CERTIFICATE FILES WITH WRONG CERTIFICATES PARSED")
-            logger.warning(broken_files)
-            logger.warning("... skipping these...")
-            logger.warning(f"Total non-analyzable files:{len(broken_files)}")
-
-        for current_cert in self.certs.values():
-            current_cert.processed.connections = []
-            if not current_cert.state.file_status or not current_cert.processed.keywords:
-                continue
-            if current_cert.processed.keywords['rules_cert_id'] == {}:
-                continue
-            for rule in current_cert.processed.keywords['rules_cert_id']:
-                for cert in current_cert.processed.keywords['rules_cert_id'][rule]:
-                    cert_id = ''.join(filter(str.isdigit, cert))
-                    if cert_id not in current_cert.processed.connections and validate_id(current_cert, cert_id):
-                        current_cert.processed.connections.append(cert_id)
-
-    def finalize_results(self):
-        self.unify_algorithms()
-        self.remove_algorithms_from_extracted_data()
-        self.validate_results()
-
-    def get_dot_graph(self, output_file_name: str):
-        """
-        Function that plots .dot graph of dependencies between certificates
-        Certificates with at least one dependency are displayed in "{output_file_name}connections.pdf", remaining
-        certificates are displayed in {output_file_name}single.pdf
-        :param output_file_name: prefix to "connections", "connections.pdf", "single" and "single.pdf"
-        """
-        dot = Digraph(comment='Certificate ecosystem')
-        single_dot = Digraph(comment='Modules with no dependencies')
-        single_dot.attr('graph', label='Single nodes', labelloc='t', fontsize='30')
-        single_dot.attr('node', style='filled')
-        dot.attr('graph', label='Dependencies', labelloc='t', fontsize='30')
-        dot.attr('node', style='filled')
-
-        def found_interesting_cert(current_key):
-            if self.certs[current_key].web_scan.vendor == highlighted_vendor:
-                dot.attr('node', color='red')
-                if self.certs[current_key].web_scan.status == 'Revoked':
-                    dot.attr('node', color='grey32')
-                if self.certs[current_key].web_scan.status == 'Historical':
-                    dot.attr('node', color='gold3')
-            if self.certs[current_key].web_scan.vendor == "SUSE, LLC":
-                dot.attr('node', color='lightblue')
-
-        def color_check(current_key):
-            dot.attr('node', color='lightgreen')
-            if self.certs[current_key].web_scan.status == 'Revoked':
-                dot.attr('node', color='lightgrey')
-            if self.certs[current_key].web_scan.status == 'Historical':
-                dot.attr('node', color='gold')
-            found_interesting_cert(current_key)
-            dot.node(current_key,
-                     label=current_key +
-                           '&#10;' +
-                           self.certs[current_key].web_scan.vendor +
-                           '&#10;' +
-                           (self.certs[current_key].web_scan.module_name
-                            if self.certs[current_key].web_scan.module_name else ''))
-
-        keys = 0
-        edges = 0
-
-        highlighted_vendor = 'Red Hat®, Inc.'
-        for key in self.certs:
-            if key != 'Not found' and self.certs[key].state.file_status:
-                if self.certs[key].processed.connections:
-                    color_check(key)
-                    keys += 1
-                else:
-                    single_dot.attr('node', color='lightblue')
-                    found_interesting_cert(key)
-                    single_dot.node(key, label=key + '\r\n' + self.certs[key].web_scan.vendor + (
-                        '\r\n' + self.certs[key].web_scan.module_name if self.certs[key].web_scan.module_name else ''))
-
-        for key in self.certs:
-            if key != 'Not found' and self.certs[key].state.file_status:
-                for conn in self.certs[key].processed.connections:
-                    color_check(conn)
-                    dot.edge(key, conn)
-                    edges += 1
-
-        logging.info(f"rendering {keys} keys and {edges} edges")
-
-        dot.render(str(output_file_name) + '_connections', view=True)
-        single_dot.render(str(output_file_name) + '_single', view=True)
-
-    def to_dict(self):
-        return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest,
-                'name': self.name, 'description': self.description,
-                'n_certs': len(self), 'certs': self.certs, 'algs': self.algorithms}
-
-    @classmethod
-    def from_dict(cls, dct: Dict):
-        certs = dct['certs']
-        dset = cls(certs, Path('./'), dct['name'], dct['description'])
-        dset.algorithms = dct['algs']
-        if len(dset) != (claimed := dct['n_certs']):
-            logger.error(
-                f'The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed}).')
-        return dset
-
-    def to_json(self, output_path: Union[str, Path]):
-        with Path(output_path).open('w') as handle:
-            json.dump(self, handle, indent=4, cls=CustomJSONEncoder)
-
-    @classmethod
-    def from_json(cls, input_path: Union[str, Path]):
-        input_path = Path(input_path)
-        with input_path.open('r') as handle:
-            dset = json.load(handle, cls=CustomJSONDecoder)
-        dset.root_dir = input_path.parent.absolute()
-        return dset
-
-    def group_vendors(self) -> Dict:
-        vendors = {}
-        v = {x.vendor.lower() for x in self.certs.values()}
-        v = sorted(v, key=FIPSCertificate.get_compare)
-        for prefix, a in groupby(v, key=FIPSCertificate.get_compare):
-            vendors[prefix] = list(a)
-
-        return vendors
-
-
-class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
-
-    def get_certs_from_web(self):
-        self.root_dir.mkdir(exist_ok=True)
-        algs_paths, algs_urls = [], []
-
-        # get first page to find out how many pages there are
-        helpers.download_file(
-            constants.FIPS_ALG_URL + '1',
-            self.root_dir / "page1.html")
-
-        with open(self.root_dir / "page1.html", "r") as alg_file:
-            soup = BeautifulSoup(alg_file.read(), 'html.parser')
-            num_pages = soup.select('span[data-total-pages]')[0].attrs
-
-        for i in range(1, int(num_pages['data-total-pages'])):
-            if not (self.root_dir / f'page{i}.html').exists():
-                algs_urls.append(
-                    constants.FIPS_ALG_URL + str(i))
-                algs_paths.append(self.root_dir / f"page{i}.html")
-
-        logging.info(f"downloading {len(algs_urls)} algs html files")
-        cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(algs_urls, algs_paths)),
-                                         constants.N_THREADS)
-
-        self.parse_html()
-
-    def parse_html(self):
-        def split_alg(alg_string):
-            cert_type = alg_string.rstrip('0123456789')
-            cert_id = alg_string[len(cert_type):]
-            return cert_type.strip(), cert_id.strip()
-
-        for f in files.search_files(self.root_dir):
-            with open(f, 'r', encoding='utf-8') as handle:
-                html_soup = BeautifulSoup(handle.read(), 'html.parser')
-
-            table = html_soup.find('table', class_='table table-condensed publications-table table-bordered')
-            spans = table.find_all('span')
-            for span in spans:
-                elements = span.find_all('td')
-                vendor, implementation = elements[0].text, elements[1].text
-                elements_sliced = elements[2:]
-                for i in range(0, len(elements_sliced), 2):
-                    alg_type, alg_id = split_alg(elements_sliced[i].text.strip())
-                    validation_date = elements_sliced[i + 1].text.strip()
-                    fips_alg = FIPSCertificate.Algorithm(alg_id, vendor, implementation, alg_type, validation_date)
-                    if alg_id not in self.certs:
-                        self.certs[alg_id] = []
-                    self.certs[alg_id].append(fips_alg)
-
-    def convert_all_pdfs(self):
-        raise NotImplementedError('Not meant to be implemented')
-
-    def download_all_pdfs(self):
-        raise NotImplementedError('Not meant to be implemented')
-
-    def to_dict(self):
-        return {"certs": self.certs}
-
-    @classmethod
-    def from_dict(cls, dct: Dict):
-        certs = dct['certs']
-        dset = cls(certs, Path('./'), 'algorithms', 'algorithms used in dataset')
-        return dset
-
-    def to_json(self, output_path: Union[str, Path]):
-        with Path(output_path).open('w') as handle:
-            json.dump(self, handle, indent=4, cls=CustomJSONEncoder)
-
-    @classmethod
-    def from_json(cls, input_path: Union[str, Path]):
-        input_path = Path(input_path)
-        with input_path.open('r') as handle:
-            dset = json.load(handle, cls=CustomJSONDecoder)
-        dset.root_dir = input_path.parent.absolute()
-        return dset
+            cert.compute_heuristics_related_cves(cve_dset)
\ No newline at end of file
diff --git a/sec_certs/cpe.py b/sec_certs/dataset/cpe.py
index e3a456f5..e3a456f5 100644
--- a/sec_certs/cpe.py
+++ b/sec_certs/dataset/cpe.py
diff --git a/sec_certs/cve.py b/sec_certs/dataset/cve.py
index a8cc9742..a8cc9742 100644
--- a/sec_certs/cve.py
+++ b/sec_certs/dataset/cve.py
diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py
new file mode 100644
index 00000000..469a8e06
--- /dev/null
+++ b/sec_certs/dataset/dataset.py
@@ -0,0 +1,123 @@
+from datetime import datetime
+import logging
+from typing import Dict, Collection, Union
+
+import json
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+import requests
+
+import sec_certs.helpers as helpers
+import sec_certs.constants as constants
+import sec_certs.cert_processing as cert_processing
+
+from sec_certs.certificate import Certificate
+from sec_certs.serialization import CustomJSONDecoder, CustomJSONEncoder
+
+logger = logging.getLogger(__name__)
+
+
+class Dataset(ABC):
+    def __init__(self, certs: Dict[str, 'Certificate'], root_dir: Path, name: str = 'dataset name',
+                 description: str = 'dataset_description'):
+        self._root_dir = root_dir
+        self.timestamp = datetime.now()
+        self.sha256_digest = 'not implemented'
+        self.name = name
+        self.description = description
+        self.certs = certs
+
+    @property
+    def root_dir(self):
+        return self._root_dir
+
+    @root_dir.setter
+    def root_dir(self, new_dir: Union[str, Path]):
+        if not (new_path := Path(new_dir)).exists():
+            raise FileNotFoundError('Root directory for Dataset does not exist')
+        self._root_dir = new_path
+
+    @property
+    def json_path(self) -> Path:
+        return self.root_dir / (self.name + '.json')
+
+    def __iter__(self):
+        yield from self.certs.values()
+
+    def __getitem__(self, item: str):
+        return self.certs.__getitem__(item.lower())
+
+    def __setitem__(self, key: str, value: 'Certificate'):
+        self.certs.__setitem__(key.lower(), value)
+
+    def __len__(self) -> int:
+        return len(self.certs)
+
+    def __eq__(self, other: 'Dataset') -> bool:
+        return self.certs == other.certs
+
+    def __str__(self) -> str:
+        return str(type(self).__name__) + ':' + self.name + ', ' + str(len(self)) + ' certificates'
+
+    def to_dict(self):
+        return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest,
+                'name': self.name, 'description': self.description,
+                'n_certs': len(self), 'certs': list(self.certs.values())}
+
+    @classmethod
+    def from_dict(cls, dct: Dict):
+        certs = {x.dgst: x for x in dct['certs']}
+        dset = cls(certs, Path('../'), dct['name'], dct['description'])
+        if len(dset) != (claimed := dct['n_certs']):
+            logger.error(
+                f'The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed}).')
+        return dset
+
+    def to_json(self, output_path: Union[str, Path] = None):
+        if not output_path:
+            output_path = self.json_path
+
+        with Path(output_path).open('w') as handle:
+            json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
+
+    @classmethod
+    def from_json(cls, input_path: Union[str, Path]):
+        input_path = Path(input_path)
+        with input_path.open('r') as handle:
+            dset = json.load(handle, cls=CustomJSONDecoder)
+        dset.root_dir = input_path.parent.absolute()
+        return dset
+
+    @abstractmethod
+    def get_certs_from_web(self):
+        raise NotImplementedError('Not meant to be implemented by the base class.')
+
+    @abstractmethod
+    def convert_all_pdfs(self):
+        raise NotImplementedError('Not meant to be implemented by the base class.')
+
+    @abstractmethod
+    def download_all_pdfs(self):
+        raise NotImplementedError('Not meant to be implemented by the base class.')
+
+    @staticmethod
+    def _download_parallel(urls: Collection[str], paths: Collection[Path], prune_corrupted: bool = True):
+        exit_codes = cert_processing.process_parallel(helpers.download_file,
+                                                      list(zip(urls, paths)),
+                                                      constants.N_THREADS,
+                                                      unpack=True)
+        n_successful = len([e for e in exit_codes if e == requests.codes.ok])
+        logger.info(f'Successfully downloaded {n_successful} files, {len(exit_codes) - n_successful} failed.')
+
+        for url, e in zip(urls, exit_codes):
+            if e != requests.codes.ok:
+                logger.error(f'Failed to download {url}, exit code: {e}')
+
+        if prune_corrupted is True:
+            for p in paths:
+                if p.exists() and p.stat().st_size < constants.MIN_CORRECT_CERT_SIZE:
+                    logger.error(f'Corrupted file at: {p}')
+                    # TODO: Delete
+
+
diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py
new file mode 100644
index 00000000..bb1112fd
--- /dev/null
+++ b/sec_certs/dataset/fips.py
@@ -0,0 +1,409 @@
+import json
+import logging
+import os
+from itertools import groupby
+from pathlib import Path
+from typing import ClassVar, Tuple, List, Dict, Optional, Union
+
+from bs4 import BeautifulSoup
+from graphviz import Digraph
+
+from sec_certs import constants as constants, cert_processing as cert_processing, helpers as helpers
+from sec_certs.configuration import config
+from sec_certs.dataset.dataset import Dataset, logger
+from sec_certs.serialization import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder
+from sec_certs.certificate.fips import FIPSCertificate
+
+
+class FIPSDataset(Dataset, ComplexSerializableType):
+    FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov'
+    FIPS_MODULE_URL: ClassVar[
+        str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
+
+    def __init__(self, certs: dict, root_dir: Path, name: str = 'dataset name',
+                 description: str = 'dataset_description'):
+        super().__init__(certs, root_dir, name, description)
+        self.keywords = {}
+        self.algorithms = None
+        self.new_files = 0
+
+    @property
+    def web_dir(self) -> Path:
+        return self.root_dir / 'web'
+
+    @property
+    def results_dir(self) -> Path:
+        return self.root_dir / 'results'
+
+    @property
+    def policies_dir(self) -> Path:
+        return self.root_dir / 'security_policies'
+
+    @property
+    def fragments_dir(self) -> Path:
+        return self.root_dir / 'fragments'
+
+    @property
+    def algs_dir(self) -> Path:
+        return self.web_dir / 'algorithms'
+
+    def find_empty_pdfs(self) -> Tuple[List, List]:
+        missing = []
+        not_available = []
+        for i in self.certs:
+            if not (self.policies_dir / f'{i}.pdf').exists():
+                missing.append(i)
+            elif os.path.getsize(self.policies_dir / f'{i}.pdf') < constants.FIPS_NOT_AVAILABLE_CERT_SIZE:
+                not_available.append(i)
+        return missing, not_available
+
+    def extract_keywords(self, redo=False):
+        self.fragments_dir.mkdir(parents=True, exist_ok=True)
+
+        keywords = cert_processing.process_parallel(FIPSCertificate.find_keywords,
+                                                    [cert for cert in self.certs.values() if
+                                                     not cert.pdf_scan.keywords or redo],
+                                                    constants.N_THREADS,
+                                                    use_threading=False)
+        for keyword, cert in keywords:
+            self.certs[cert.dgst].pdf_scan.keywords = keyword
+
+    def match_algs(self, show_graph=False) -> Dict:
+        output = {}
+        for cert in self.certs.values():
+            output[cert.dgst] = FIPSCertificate.match_web_algs_to_pdf(cert)
+
+        return output
+
+
+    def download_all_pdfs(self):
+        sp_paths, sp_urls = [], []
+        self.policies_dir.mkdir(exist_ok=True)
+
+        for cert_id in list(self.certs.keys()):
+            if not (self.policies_dir / f'{cert_id}.pdf').exists() or not self.certs[cert_id].state.txt_state:
+                sp_urls.append(
+                    f"https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp{cert_id}.pdf")
+                sp_paths.append(self.policies_dir / f"{cert_id}.pdf")
+        logging.info(f"downloading {len(sp_urls)} module pdf files")
+        cert_processing.process_parallel(FIPSCertificate.download_security_policy, list(zip(sp_urls, sp_paths)),
+                                         constants.N_THREADS)
+        self.new_files += len(sp_urls)
+
+    def download_all_htmls(self) -> List[str]:
+        html_paths, html_urls = [], []
+        new_files = []
+        self.web_dir.mkdir(exist_ok=True)
+        for cert_id in self.certs.keys():
+            if not (self.web_dir / f'{cert_id}.html').exists():
+                html_urls.append(
+                    f"https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{cert_id}")
+                html_paths.append(self.web_dir / f"{cert_id}.html")
+                new_files.append(cert_id)
+
+        logging.info(f"downloading {len(html_urls)} module html files")
+        failed = cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(html_urls, html_paths)),
+                                         constants.N_THREADS)
+        failed = [c for c in failed if c]
+
+        self.new_files += len(html_urls)
+        logging.info(f"Download failed for {len(failed)} files. Retrying...")
+        cert_processing.process_parallel(FIPSCertificate.download_html_page, failed,
+                                         constants.N_THREADS)
+        return new_files
+
+    def convert_all_pdfs(self):
+        logger.info('Converting FIPS certificate reports to .txt')
+        tuples = [
+            (cert, self.policies_dir / f'{cert.cert_id}.pdf', self.policies_dir / f'{cert.cert_id}.pdf.txt')
+            for cert in self.certs.values() if
+            not cert.state.txt_state and (self.policies_dir / f'{cert.cert_id}.pdf').exists()
+        ]
+        cert_processing.process_parallel(FIPSCertificate.convert_pdf_file, tuples, constants.N_THREADS)
+
+    def get_certs_from_web(self, redo: bool = False, json_file: Optional[Path] = None):
+        def download_html_pages() -> List[str]:
+            new_files = self.download_all_htmls()
+            self.download_all_pdfs()
+            return new_files
+
+        def get_certificates_from_html(html_file: Path) -> None:
+            logger.info(f'Getting certificate ids from {html_file}')
+            with open(html_file, 'r', encoding='utf-8') as handle:
+                html = BeautifulSoup(handle.read(), 'html.parser')
+
+            table = [x for x in html.find(
+                id='searchResultsTable').tbody.contents if x != '\n']
+            for entry in table:
+                self.certs[entry.find('a').text] = {}
+
+        logger.info("Downloading required html files")
+
+        self.web_dir.mkdir(parents=True, exist_ok=True)
+        self.policies_dir.mkdir(exist_ok=True)
+        self.algs_dir.mkdir(exist_ok=True)
+
+        # Download files containing all available module certs (always)
+        html_files = ['fips_modules_active.html',
+                      'fips_modules_historical.html', 'fips_modules_revoked.html']
+        helpers.download_file(
+            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Active&ValidationYear=0",
+            self.web_dir / "fips_modules_active.html")
+        helpers.download_file(
+            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Historical&ValidationYear=0",
+            self.web_dir / "fips_modules_historical.html")
+        helpers.download_file(
+            "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search?SearchMode=Advanced&CertificateStatus=Revoked&ValidationYear=0",
+            self.web_dir / "fips_modules_revoked.html")
+
+        # Parse those files and get list of currently processable files (always)
+        for f in html_files:
+            get_certificates_from_html(self.web_dir / f)
+
+        logger.info('Downloading certificate html and security policies')
+
+        if not json_file:
+            json_file = self.root_dir / 'fips_full_dataset.json'
+
+        if json_file.exists():
+            logger.info("Certs loaded from previous scanning")
+            dataset = self.from_json(json_file)
+            self.certs = dataset.certs
+            self.algorithms = dataset.algorithms
+
+        new_certs = download_html_pages()
+
+        logger.info(f"{self.new_files} needed to be downloaded")
+
+        for cert_id in new_certs:
+            self.certs[cert_id] = None
+
+        if not redo and self.new_files == 0:
+            logger.info('No new changes to web_scan are going to be made')
+            return
+        # now we want to do redo, because we want to avoid duplicites
+        redo = True
+        logger.info(f'Parsing web pages{" from scratch" if redo else ""}...')
+        for cert_id, cert in self.certs.items():
+            self.certs[cert_id] = FIPSCertificate.html_from_file(
+                self.web_dir / f'{cert_id}.html',
+                FIPSCertificate.State((self.policies_dir / cert_id).with_suffix('.pdf'),
+                                      (self.web_dir / cert_id).with_suffix('.html'),
+                                      (self.fragments_dir / cert_id).with_suffix('.txt'), False, None, False),
+                cert, redo=redo)
+
+    def extract_certs_from_tables(self) -> List[Path]:
+        """
+        Function that extracts algorithm IDs from tables in security policies files.
+        :return: list of files that couldn't have been decoded
+        """
+        result = cert_processing.process_parallel(FIPSCertificate.analyze_tables,
+                                                  [cert for cert in self.certs.values() if
+                                                   not cert.state.tables_done and cert.state.txt_state],
+                                                  constants.N_THREADS // 4,  # tabula already processes by parallel, so
+                                                  # it's counterproductive to use all threads
+                                                  use_threading=False)
+
+        not_decoded = [cert.state.sp_path for done, cert, _ in result if done is False]
+        for state, cert, algorithms in result:
+            self.certs[cert.dgst].state.tables_done = state
+            self.certs[cert.dgst].pdf_scan.algorithms += algorithms
+
+        return not_decoded
+
+    def remove_algorithms_from_extracted_data(self):
+        for cert in self.certs.values():
+            cert.remove_algorithms()
+
+    def unify_algorithms(self):
+        certificate: FIPSCertificate
+        for certificate in self.certs.values():
+            new_algorithms = []
+            united_algorithms = [x for x in (certificate.web_scan.algorithms + certificate.pdf_scan.algorithms) if
+                                 x != {'Certificate': []}]
+            for algorithm in united_algorithms:
+                if isinstance(algorithm, dict):
+                    new_algorithms.append(algorithm)
+                else:
+                    new_algorithms.append({'Certificate': [algorithm]})
+            certificate.processed.algorithms = new_algorithms
+
+    def validate_results(self):
+        """
+        Function that validates results and finds the final connection output
+        """
+
+        def validate_id(processed_cert: FIPSCertificate, cert_candidate: str) -> bool:
+
+            # returns True if candidates should _not_ be matched
+            def compare_certs(current_certificate: 'FIPSCertificate', other_id: str):
+                cert_first = current_certificate.web_scan.date_validation[0].year
+                cert_last = current_certificate.web_scan.date_validation[-1].year
+                conn_first = self.certs[other_id].web_scan.date_validation[0].year
+                conn_last = self.certs[other_id].web_scan.date_validation[-1].year
+
+                return cert_first - conn_first > config.year_difference_between_validations['value'] \
+                       and cert_last - conn_last > config.year_difference_between_validations['value'] \
+                       or cert_first < conn_first
+
+            # "< number" still needs to be used, because of some old certs being revalidated
+            if cert_candidate.isdecimal() \
+                    and int(cert_candidate) < config.smallest_certificate_id_to_connect['value'] or \
+                    compare_certs(processed_cert, cert_candidate):
+                return False
+            if cert_candidate not in self.algorithms.certs:
+                return True
+
+            for cert_alg in processed_cert.processed.algorithms:
+                for certificate in cert_alg['Certificate']:
+                    curr_id = ''.join(filter(str.isdigit, certificate))
+                    if curr_id == cert_candidate:
+                        return False
+
+            algs = self.algorithms.certs[cert_candidate]
+            for current_alg in algs:
+                if FIPSCertificate.get_compare(processed_cert.web_scan.vendor) == FIPSCertificate.get_compare(
+                        current_alg.vendor):
+                    return False
+            return True
+
+        broken_files = set()
+
+        current_cert: FIPSCertificate
+
+        for current_cert in self.certs.values():
+            if not current_cert.state.txt_state:
+                continue
+            for rule in current_cert.processed.keywords['rules_cert_id']:
+                for cert in current_cert.processed.keywords['rules_cert_id'][rule]:
+                    cert_id = ''.join(filter(str.isdigit, cert))
+
+                    if cert_id == '' or cert_id not in self.certs:
+                        broken_files.add(current_cert.dgst)
+                        current_cert.state.file_status = False
+                        break
+
+        if broken_files:
+            logger.warning("CERTIFICATE FILES WITH WRONG CERTIFICATES PARSED")
+            logger.warning(broken_files)
+            logger.warning("... skipping these...")
+            logger.warning(f"Total non-analyzable files:{len(broken_files)}")
+
+        for current_cert in self.certs.values():
+            current_cert.processed.connections = []
+            if not current_cert.state.file_status or not current_cert.processed.keywords:
+                continue
+            if current_cert.processed.keywords['rules_cert_id'] == {}:
+                continue
+            for rule in current_cert.processed.keywords['rules_cert_id']:
+                for cert in current_cert.processed.keywords['rules_cert_id'][rule]:
+                    cert_id = ''.join(filter(str.isdigit, cert))
+                    if cert_id not in current_cert.processed.connections and validate_id(current_cert, cert_id):
+                        current_cert.processed.connections.append(cert_id)
+
+    def finalize_results(self):
+        self.unify_algorithms()
+        self.remove_algorithms_from_extracted_data()
+        self.validate_results()
+
+    def get_dot_graph(self, output_file_name: str):
+        """
+        Function that plots .dot graph of dependencies between certificates
+        Certificates with at least one dependency are displayed in "{output_file_name}connections.pdf", remaining
+        certificates are displayed in {output_file_name}single.pdf
+        :param output_file_name: prefix to "connections", "connections.pdf", "single" and "single.pdf"
+        """
+        dot = Digraph(comment='Certificate ecosystem')
+        single_dot = Digraph(comment='Modules with no dependencies')
+        single_dot.attr('graph', label='Single nodes', labelloc='t', fontsize='30')
+        single_dot.attr('node', style='filled')
+        dot.attr('graph', label='Dependencies', labelloc='t', fontsize='30')
+        dot.attr('node', style='filled')
+
+        def found_interesting_cert(current_key):
+            if self.certs[current_key].web_scan.vendor == highlighted_vendor:
+                dot.attr('node', color='red')
+                if self.certs[current_key].web_scan.status == 'Revoked':
+                    dot.attr('node', color='grey32')
+                if self.certs[current_key].web_scan.status == 'Historical':
+                    dot.attr('node', color='gold3')
+            if self.certs[current_key].web_scan.vendor == "SUSE, LLC":
+                dot.attr('node', color='lightblue')
+
+        def color_check(current_key):
+            dot.attr('node', color='lightgreen')
+            if self.certs[current_key].web_scan.status == 'Revoked':
+                dot.attr('node', color='lightgrey')
+            if self.certs[current_key].web_scan.status == 'Historical':
+                dot.attr('node', color='gold')
+            found_interesting_cert(current_key)
+            dot.node(current_key,
+                     label=current_key +
+                           '&#10;' +
+                           self.certs[current_key].web_scan.vendor +
+                           '&#10;' +
+                           (self.certs[current_key].web_scan.module_name
+                            if self.certs[current_key].web_scan.module_name else ''))
+
+        keys = 0
+        edges = 0
+
+        highlighted_vendor = 'Red Hat®, Inc.'
+        for key in self.certs:
+            if key != 'Not found' and self.certs[key].state.file_status:
+                if self.certs[key].processed.connections:
+                    color_check(key)
+                    keys += 1
+                else:
+                    single_dot.attr('node', color='lightblue')
+                    found_interesting_cert(key)
+                    single_dot.node(key, label=key + '\r\n' + self.certs[key].web_scan.vendor + (
+                        '\r\n' + self.certs[key].web_scan.module_name if self.certs[key].web_scan.module_name else ''))
+
+        for key in self.certs:
+            if key != 'Not found' and self.certs[key].state.file_status:
+                for conn in self.certs[key].processed.connections:
+                    color_check(conn)
+                    dot.edge(key, conn)
+                    edges += 1
+
+        logging.info(f"rendering {keys} keys and {edges} edges")
+
+        dot.render(str(output_file_name) + '_connections', view=True)
+        single_dot.render(str(output_file_name) + '_single', view=True)
+
+    def to_dict(self):
+        return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest,
+                'name': self.name, 'description': self.description,
+                'n_certs': len(self), 'certs': self.certs, 'algs': self.algorithms}
+
+    @classmethod
+    def from_dict(cls, dct: Dict):
+        certs = dct['certs']
+        dset = cls(certs, Path('../'), dct['name'], dct['description'])
+        dset.algorithms = dct['algs']
+        if len(dset) != (claimed := dct['n_certs']):
+            logger.error(
+                f'The actual number of certs in dataset ({len(dset)}) does not match the claimed number ({claimed}).')
+        return dset
+
+    def to_json(self, output_path: Union[str, Path]):
+        with Path(output_path).open('w') as handle:
+            json.dump(self, handle, indent=4, cls=CustomJSONEncoder)
+
+    @classmethod
+    def from_json(cls, input_path: Union[str, Path]):
+        input_path = Path(input_path)
+        with input_path.open('r') as handle:
+            dset = json.load(handle, cls=CustomJSONDecoder)
+        dset.root_dir = input_path.parent.absolute()
+        return dset
+
+    def group_vendors(self) -> Dict:
+        vendors = {}
+        v = {x.vendor.lower() for x in self.certs.values()}
+        v = sorted(v, key=FIPSCertificate.get_compare)
+        for prefix, a in groupby(v, key=FIPSCertificate.get_compare):
+            vendors[prefix] = list(a)
+
+        return vendors
\ No newline at end of file
diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py
new file mode 100644
index 00000000..a89637c1
--- /dev/null
+++ b/sec_certs/dataset/fips_algorithm.py
@@ -0,0 +1,89 @@
+import json
+import logging
+from pathlib import Path
+from typing import Dict, Union
+
+from bs4 import BeautifulSoup
+
+from sec_certs import helpers as helpers, constants as constants, cert_processing as cert_processing, files as files
+from sec_certs.dataset.dataset import Dataset
+from sec_certs.serialization import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder
+from sec_certs.certificate.fips import FIPSCertificate
+
+class FIPSAlgorithmDataset(Dataset, ComplexSerializableType):
+
+    def get_certs_from_web(self):
+        self.root_dir.mkdir(exist_ok=True)
+        algs_paths, algs_urls = [], []
+
+        # get first page to find out how many pages there are
+        helpers.download_file(
+            constants.FIPS_ALG_URL + '1',
+            self.root_dir / "page1.html")
+
+        with open(self.root_dir / "page1.html", "r") as alg_file:
+            soup = BeautifulSoup(alg_file.read(), 'html.parser')
+            num_pages = soup.select('span[data-total-pages]')[0].attrs
+
+        for i in range(1, int(num_pages['data-total-pages'])):
+            if not (self.root_dir / f'page{i}.html').exists():
+                algs_urls.append(
+                    constants.FIPS_ALG_URL + str(i))
+                algs_paths.append(self.root_dir / f"page{i}.html")
+
+        logging.info(f"downloading {len(algs_urls)} algs html files")
+        cert_processing.process_parallel(FIPSCertificate.download_html_page, list(zip(algs_urls, algs_paths)),
+                                         constants.N_THREADS)
+
+        self.parse_html()
+
+    def parse_html(self):
+        def split_alg(alg_string):
+            cert_type = alg_string.rstrip('0123456789')
+            cert_id = alg_string[len(cert_type):]
+            return cert_type.strip(), cert_id.strip()
+
+        for f in files.search_files(self.root_dir):
+            with open(f, 'r', encoding='utf-8') as handle:
+                html_soup = BeautifulSoup(handle.read(), 'html.parser')
+
+            table = html_soup.find('table', class_='table table-condensed publications-table table-bordered')
+            spans = table.find_all('span')
+            for span in spans:
+                elements = span.find_all('td')
+                vendor, implementation = elements[0].text, elements[1].text
+                elements_sliced = elements[2:]
+                for i in range(0, len(elements_sliced), 2):
+                    alg_type, alg_id = split_alg(elements_sliced[i].text.strip())
+                    validation_date = elements_sliced[i + 1].text.strip()
+                    fips_alg = FIPSCertificate.Algorithm(alg_id, vendor, implementation, alg_type, validation_date)
+                    if alg_id not in self.certs:
+                        self.certs[alg_id] = []
+                    self.certs[alg_id].append(fips_alg)
+
+    def convert_all_pdfs(self):
+        raise NotImplementedError('Not meant to be implemented')
+
+    def download_all_pdfs(self):
+        raise NotImplementedError('Not meant to be implemented')
+
+    def to_dict(self):
+        return {"certs": self.certs}
+
+    @classmethod
+    def from_dict(cls, dct: Dict):
+        certs = dct['certs']
+        dset = cls(certs, Path('../'), 'algorithms', 'algorithms used in dataset')
+        return dset
+
+    def to_json(self, output_path: Union[str, Path]):
+        with Path(output_path).open('w') as handle:
+            json.dump(self, handle, indent=4, cls=CustomJSONEncoder)
+
+    @classmethod
+    def from_json(cls, input_path: Union[str, Path]):
+        input_path = Path(input_path)
+        with input_path.open('r') as handle:
+            dset = json.load(handle, cls=CustomJSONDecoder)
+        dset.root_dir = input_path.parent.absolute()
+        return dset
\ No newline at end of file