Rethought way of processing certificates

- Every certificate now implements a static method for its processing
- These are called in parallel
- There's some code duplication at the moment, stuff can be simplified.

8 files changed, 252 insertions, 100 deletions

diff --git a/cc_oop_demo.py b/cc_oop_demo.py
index c78759df..1da4cfe8 100644
--- a/cc_oop_demo.py
+++ b/cc_oop_demo.py
@@ -25,11 +25,11 @@ def main():
     dset.get_certs_from_web()
     logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.')
 
-    # Dump dataset into JSON
+    # # Dump dataset into JSON
     dset.to_json('./debug_dataset/cc_full_dataset.json')
 
     # Load dataset from JSON
-    new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json')
+    new_dset = CCDataset.from_json('/Users/adam/phd/projects/certificates/sec-certs/debug_dataset/cc_full_dataset.json')
 
     assert dset == new_dset
 
@@ -37,7 +37,7 @@ def main():
     dset.download_all_pdfs()
 
     # Convert pdfs to text
-    new_dset.convert_all_pdfs()
+    dset.convert_all_pdfs()
 
     end = datetime.now()
     logger.info(f'The computation took {(end-start)} seconds.')
diff --git a/sec_certs/cert_processing.py b/sec_certs/cert_processing.py
index 0ccac952..739bfa9f 100644
--- a/sec_certs/cert_processing.py
+++ b/sec_certs/cert_processing.py
@@ -11,7 +11,7 @@ def process_parallel(func: Callable, items: Iterable, max_workers: int, callback
     else:
         pool = Pool(max_workers)
 
-    results = [pool.apply_async(func, (*i, ), callback=callback) for i in items]
+    results = [pool.apply_async(func, (i, ), callback=callback) for i in items]
 
     if progress_bar is True:
         bar = tqdm(total=len(results))
@@ -25,4 +25,4 @@ def process_parallel(func: Callable, items: Iterable, max_workers: int, callback
     pool.close()
     pool.join()
 
-    return [r.get() for r in results]
\ No newline at end of file
+    return [r.get() for r in results]
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index a9ce04b7..9f16ab42 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -6,6 +6,7 @@ from pathlib import Path
 import os
 import copy
 import json
+import requests
 
 from abc import ABC, abstractmethod
 from bs4 import Tag, BeautifulSoup, NavigableString
@@ -13,6 +14,7 @@ from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type
 
 from sec_certs import helpers, extract_certificates
 from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
+import sec_certs.constants as constants
 
 logger = logging.getLogger(__name__)
 
@@ -249,7 +251,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType):
                 'div', 'panel-body').children)[2].strip().split('\n')[1].strip()
 
         if html_items_found['fips_lab'] == '':
-            loggerr.warning(f"WARNING: NO LAB FOUND{current_file}")
+            logger.warning(f"WARNING: NO LAB FOUND{current_file}")
 
         if html_items_found['fips_nvlap_code'] == '':
             logger.warning(f"WARNING: NO NVLAP CODE FOUND{current_file}")
@@ -389,12 +391,39 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         def __lt__(self, other):
             return self.pp_name < other.pp_name
 
+    @dataclass(init=False)
+    class InternalState(ComplexSerializableType):
+        st_link_ok: bool
+        report_link_ok: bool
+        st_convert_ok: bool
+        report_convert_ok: bool
+        st_pdf_path: Path
+        report_pdf_path: Path
+        st_txt_path: Path
+        report_txt_path: Path
+
+        def __init__(self, st_link_ok: bool = True, report_link_ok: bool = True,
+                     st_convert_ok: bool = True, report_convert_ok: bool = True):
+            self.st_link_ok = st_link_ok
+            self.report_link_ok = report_link_ok
+            self.st_convert_ok = st_convert_ok
+            self.report_convert_ok = report_convert_ok
+
+        def to_dict(self):
+            return {'st_link_ok': self.st_link_ok, 'report_link_ok': self.report_link_ok,
+                    'st_convert_ok': self.st_convert_ok, 'report_convert_ok': self.report_convert_ok}
+
+        @classmethod
+        def from_dict(cls, dct: Dict[str, bool]):
+            return cls(*tuple(dct.values()))
+
     def __init__(self, category: str, name: str, manufacturer: str, scheme: str,
                  security_level: Union[str, set], not_valid_before: date,
                  not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str],
                  manufacturer_web: Optional[str],
                  protection_profiles: set,
-                 maintainance_updates: set):
+                 maintainance_updates: set,
+                 state: Optional[InternalState]):
         super().__init__()
 
         self.category = category
@@ -412,8 +441,10 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
         self.protection_profiles = protection_profiles
         self.maintainance_updates = maintainance_updates
 
-        if self.st_link == self.empty_st_url:
-            self.st_link = None
+        if state is not None:
+            self.state = state
+        else:
+            self.state = self.InternalState()
 
     @property
     def dgst(self) -> str:
@@ -442,6 +473,8 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
                 setattr(self, att, getattr(other, att))
             elif att == 'src':
                 pass  # This is expected
+            elif att == 'state':
+                setattr(self, att, getattr(other, att))
             else:
                 if getattr(self, att) != getattr(other, att):
                     logger.warning(
@@ -566,4 +599,53 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
             maintainance_div) if maintainance_div else set()
 
         return cls(category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, report_link,
-                   st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances)
+                   st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None)
+
+    def set_local_paths(self,
+                        report_pdf_dir: Optional[Union[str, Path]],
+                        st_pdf_dir: Optional[Union[str, Path]],
+                        report_txt_dir: Optional[Union[str, Path]],
+                        st_txt_dir: Optional[Union[str, Path]]):
+        if report_pdf_dir is not None:
+            self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + '.pdf')
+        if st_pdf_dir is not None:
+            self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + '.pdf')
+        if report_txt_dir is not None:
+            self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + '.txt')
+        if st_txt_dir is not None:
+            self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + '.txt')
+
+    @staticmethod
+    def download_pdf_report(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path)
+        if exit_code != requests.codes.ok:
+            logger.error(f'Failed to download report from {cert.report_link}, code: {exit_code}')
+            cert.state.report_link_ok = False
+        return cert
+
+    @staticmethod
+    def download_pdf_target(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path)
+        if exit_code != requests.codes.ok:
+            logger.error(f'Cert dgst: {cert.dgst} failed to download report from {cert.report_link}, code: {exit_code}')
+            cert.state.st_link_ok = False
+        return cert
+
+    def path_is_corrupted(self, local_path):
+        return local_path.exists() and local_path.stat().st_size >= constants.MIN_CORRECT_CERT_SIZE
+
+    @staticmethod
+    def convert_report_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path, ['-raw'])
+        if exit_code != constants.RETURNCODE_OK:
+            logger.error(f'Cert dgst: {cert.dgst} failed to convert report pdf->txt')
+            cert.state.report_convert_ok = False
+        return cert
+
+    @staticmethod
+    def convert_target_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path, ['-raw'])
+        if exit_code != constants.RETURNCODE_OK:
+            logger.error(f'Cert dgst: {cert.dgst} failed to convert security target pdf->txt')
+            cert.state.st_convert_ok = False
+        return cert
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 22ccfa6b..56466e2e 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -3,7 +3,7 @@ import re
 from datetime import datetime
 import locale
 import logging
-from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union, Generic, Optional, Sequence
+from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union, Generic, Optional, Sequence, Tuple
 import json
 from importlib import import_module
 
@@ -37,13 +37,23 @@ logger = logging.getLogger(__name__)
 class Dataset(ABC):
     def __init__(self, certs: Dict[str, 'Certificate'], root_dir: Path, name: str = 'dataset name',
                  description: str = 'dataset_description'):
-        self.root_dir = root_dir
+        self._root_dir = root_dir
         self.timestamp = datetime.now()
         self.sha256_digest = 'not implemented'
         self.name = name
         self.description = description
         self.certs = certs
 
+    @property
+    def root_dir(self):
+        return self._root_dir
+
+    @root_dir.setter
+    def root_dir(self, new_dir: Union[str, Path]):
+        if not Path(new_dir).exists():
+            raise FileNotFoundError('Root directory for Dataset does not exist')
+        self._root_dir = Path(new_dir)
+
     def __iter__(self):
         for cert in self.certs.values():
             yield cert
@@ -81,9 +91,10 @@ class Dataset(ABC):
 
     @classmethod
     def from_json(cls, input_path: Union[str, Path]):
-        with Path(input_path).open('r') as handle:
+        input_path = Path(input_path)
+        with input_path.open('r') as handle:
             dset = json.load(handle, cls=CustomJSONDecoder)
-        dset.root_path = input_path.parent
+        dset.root_dir = input_path.parent.absolute()
         return dset
 
     @abstractmethod
@@ -99,25 +110,8 @@ class Dataset(ABC):
         raise NotImplementedError('Not meant to be implemented by the base class.')
 
     @staticmethod
-    def _convert_pdfs_to_txt(pdf_paths: Collection[Path], txt_paths: Collection[Path]):
-        assert len(pdf_paths) == len(txt_paths)
-
-        partial_convert_pdf = partial(helpers.convert_pdf_file, options=['-raw'])
-        exit_codes = cert_processing.process_parallel(partial_convert_pdf,
-                                                      list(zip(pdf_paths, txt_paths)),
-                                                      constants.N_THREADS,
-                                                      use_threading=False)
-
-        n_successful = len([e for e in exit_codes if e == constants.RETURNCODE_OK])
-        logger.info(f'Successfully converted {n_successful} files pdf->txt, {len(exit_codes) - n_successful} failed.')
-
-        for path, e in zip(pdf_paths, exit_codes):
-            if e != constants.RETURNCODE_OK:
-                logger.info(f'Failed to convert {path}, exit code: {e}')
-
-    @staticmethod
     def _download_parallel(urls: Collection[str], paths: Collection[Path], prune_corrupted: bool = True):
-        exit_codes = cert_processing.process_parallel(download.download_file,
+        exit_codes = cert_processing.process_parallel(helpers.download_file,
                                                       list(zip(urls, paths)),
                                                       constants.N_THREADS)
         n_successful = len([e for e in exit_codes if e == requests.codes.ok])
@@ -135,6 +129,13 @@ class Dataset(ABC):
 
 
 class CCDataset(Dataset, ComplexSerializableType):
+    # TODO: Make properties propagate to changing internal state of related certificates
+
+    @Dataset.root_dir.setter
+    def root_dir(self, new_dir: Union[str, Path]):
+        Dataset.root_dir.fset(self, new_dir)
+        self.set_local_paths()
+
     @property
     def web_dir(self) -> Path:
         return self.root_dir / 'web'
@@ -167,22 +168,6 @@ class CCDataset(Dataset, ComplexSerializableType):
     def targets_txt_dir(self) -> Path:
         return self.targets_dir / 'txt'
 
-    @property
-    def report_pdf_paths(self) -> Dict[str, Path]:
-        return {x: self.reports_pdf_dir / (self[x].dgst + '.pdf') for x in self.certs}
-
-    @property
-    def report_txt_paths(self) -> Dict[str, Path]:
-        return {x: self.reports_txt_dir / (self[x].dgst + '.txt') for x in self.certs}
-
-    @property
-    def target_pdf_paths(self) -> Dict[str, Path]:
-        return {x: self.targets_pdf_dir / (self[x].dgst + '.pdf') for x in self.certs}
-
-    @property
-    def target_txt_paths(self) -> Dict[str, Path]:
-        return {x: self.targets_txt_dir / (self[x].dgst + '.txt') for x in self.certs}
-
     html_products = {
         'cc_products_active.html': 'https://www.commoncriteriaportal.org/products/',
         'cc_products_archived.html': 'https://www.commoncriteriaportal.org/products/index.cfm?archived=1',
@@ -202,6 +187,16 @@ class CCDataset(Dataset, ComplexSerializableType):
         'cc_pp_archived.csv': 'https://www.commoncriteriaportal.org/pps/pps-archived.csv'
     }
 
+    @classmethod
+    def from_json(cls, input_path: Union[str, Path]):
+        dset = super().from_json(input_path)
+        dset.set_local_paths()
+        return dset
+
+    def set_local_paths(self):
+        for cert in self:
+            cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir)
+
     def _merge_certs(self, certs: Dict[str, 'CommonCriteriaCert']):
         """
         Merges dictionary of certificates into the dataset. Assuming they all are CommonCriteria certificates
@@ -261,6 +256,8 @@ class CCDataset(Dataset, ComplexSerializableType):
         if not keep_metadata:
             shutil.rmtree(self.web_dir)
 
+        self.set_local_paths()
+
     def _get_all_certs_from_csv(self, get_active: bool, get_archived: bool) -> Dict[str, 'CommonCriteriaCert']:
         """
         Creates dictionary of new certificates from csv sources.
@@ -326,7 +323,7 @@ class CCDataset(Dataset, ComplexSerializableType):
 
         certs = {x.dgst: CommonCriteriaCert(x.category, x.cert_name, x.manufacturer, x.scheme, x.security_level,
                                             x.not_valid_before, x.not_valid_after, x.report_link, x.st_link, 'csv',
-                                            None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None)) for x in
+                                            None, None, profiles.get(x.dgst, None), updates.get(x.dgst, None), None) for x in
                  df_base.itertuples()}
         return certs
 
@@ -412,41 +409,74 @@ class CCDataset(Dataset, ComplexSerializableType):
 
         return certs
 
-    def _download_reports(self):
+    def _download_reports(self, fresh=True):
         self.reports_pdf_dir.mkdir(parents=True, exist_ok=True)
-        reports_urls = [x.report_link for x in self]
-        # for noqa below, see: https://youtrack.jetbrains.com/issue/PY-41771
-        self._download_parallel(reports_urls, self.report_pdf_paths.values(), prune_corrupted=True) # noqa
 
-    def _download_targets(self):
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.report_link_ok]
+
+        cert_processing.process_parallel(CommonCriteriaCert.download_pdf_report, certs_to_process, constants.N_THREADS)
+
+    def _download_targets(self, fresh=True):
         self.targets_pdf_dir.mkdir(parents=True, exist_ok=True)
-        target_urls = [x.st_link for x in self]
-        # for noqa below, see: https://youtrack.jetbrains.com/issue/PY-41771
-        self._download_parallel(target_urls, self.target_pdf_paths.values(), prune_corrupted=True) # noqa
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.st_link_ok]
+        cert_processing.process_parallel(CommonCriteriaCert.download_pdf_target, certs_to_process, constants.N_THREADS)
 
-    def download_all_pdfs(self):
+    def download_all_pdfs(self, fresh: bool = True):
         logger.info('Downloading CC certificate reports')
-        self._download_reports()
+        self._download_reports(fresh)
 
         logger.info('Downloading CC security targets')
-        self._download_targets()
+        self._download_targets(fresh)
+
+        if fresh is True:
+            # Attempt to re-download once
+            # TODO: Re-write the list comprehensions with filter?
+            if [x for x in self.certs.values() if not x.state.report_link_ok]:
+                logger.info('Attempting to re-download failed report links.')
+                self._download_reports(False)
+
+            if [x for x in self.certs.values() if not x.state.st_link_ok]:
+                logger.info('Attempting to re-download failed security target links.')
+                self._download_targets(False)
 
-    def _convert_reports_to_txt(self):
+    def _convert_reports_to_txt(self, fresh: bool = True):
         self.reports_txt_dir.mkdir(parents=True, exist_ok=True)
-        # TODO: Get rid of the list() invocation here.
-        self._convert_pdfs_to_txt(list(self.report_pdf_paths.values()), list(self.report_txt_paths.values()))
 
-    def _convert_targets_to_txt(self):
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.report_convert_ok]
+        cert_processing.process_parallel(CommonCriteriaCert.convert_report_pdf, certs_to_process, constants.N_THREADS)
+
+    def _convert_targets_to_txt(self, fresh: bool = True):
         self.targets_txt_dir.mkdir(parents=True, exist_ok=True)
-        # TODO: Get rid of the list() invocation here.
-        self._convert_pdfs_to_txt(list(self.target_pdf_paths.values()), list(self.target_txt_paths.values()))
 
-    def convert_all_pdfs(self):
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.st_convert_ok]
+        cert_processing.process_parallel(CommonCriteriaCert.convert_target_pdf, certs_to_process, constants.N_THREADS)
+
+    def convert_all_pdfs(self, fresh: bool = True):
         logger.info('Converting CC certificate reports to .txt')
-        self._convert_reports_to_txt()
+        self._convert_reports_to_txt(fresh)
 
         logger.info('Converting CC security targets to .txt')
-        self._convert_targets_to_txt()
+        self._convert_targets_to_txt(fresh)
+
+        if fresh is True:
+            if [x for x in self.certs.values() if not x.state.report_convert_ok]:
+                logger.info('Attempting to re-convert failed report pdfs')
+                self._convert_reports_to_txt(False)
+            if [x for x in self.certs.values() if not x.state.st_convert_ok]:
+                logger.info('Attempting to re-convert failed target pdfs')
+                self._convert_targets_to_txt(False)
 
 
 class FIPSDataset(Dataset, ComplexSerializableType):
diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py
index 5219a897..80e490de 100644
--- a/sec_certs/helpers.py
+++ b/sec_certs/helpers.py
@@ -13,9 +13,24 @@ from datetime import date
 import numpy as np
 import pandas as pd
 import subprocess
+import functools
 
 logger = logging.getLogger(__name__)
 
+# Following two functions are from: https://stackoverflow.com/questions/31174295/getattr-and-setattr-on-nested-subobjects-chained-properties
+
+
+def rsetattr(obj, attr, val):
+    pre, _, post = attr.rpartition('.')
+    return setattr(rgetattr(obj, pre) if pre else obj, post, val)
+
+
+def rgetattr(obj, attr, *args):
+    def _getattr(obj, attr):
+        return getattr(obj, attr, *args)
+    return functools.reduce(_getattr, [obj] + attr.split('.'))
+
+
 def download_file(url: str, output: Path) -> int:
     try:
         r = requests.get(url, allow_redirects=True, timeout=5)
diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json
index 1aeaa0db..43e8e4d1 100644
--- a/test/data/test_cc_oop/fictional_cert.json
+++ b/test/data/test_cc_oop/fictional_cert.json
@@ -29,5 +29,12 @@
             "maintainance_report_link": "https://maintainance.up",
             "maintainance_st_link": "https://maintainance.up"
         }
-    ]
+    ],
+    "state": {
+        "_type": "InternalState",
+        "st_link_ok": true,
+        "report_link_ok": true,
+        "st_convert_ok": true,
+        "report_convert_ok": true
+    }
 }
\ No newline at end of file
diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json
index a16be012..ae98c9d1 100644
--- a/test/data/test_cc_oop/toy_dataset.json
+++ b/test/data/test_cc_oop/toy_dataset.json
@@ -24,7 +24,14 @@
             "cert_link": "http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
             "manufacturer_web": "https://www.netiq.com/",
             "protection_profiles": [],
-            "maintainance_updates": []
+            "maintainance_updates": [],
+            "state": {
+                "_type": "InternalState",
+                "st_link_ok": true,
+                "report_link_ok": true,
+                "st_convert_ok": true,
+                "report_convert_ok": true
+            }
         },
         {
             "_type": "CommonCriteriaCert",
@@ -47,7 +54,14 @@
                     "pp_link": "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf"
                 }
             ],
-            "maintainance_updates": []
+            "maintainance_updates": [],
+            "state": {
+                "_type": "InternalState",
+                "st_link_ok": true,
+                "report_link_ok": true,
+                "st_convert_ok": true,
+                "report_convert_ok": true
+            }
         }
     ]
 }
\ No newline at end of file
diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py
index 98c11a4e..22c00929 100644
--- a/test/test_cc_oop.py
+++ b/test/test_cc_oop.py
@@ -8,7 +8,6 @@ import shutil
 import os
 
 from sec_certs.dataset import CCDataset
-from sec_certs.serialization import CustomJSONEncoder, CustomJSONDecoder
 from sec_certs.certificate import CommonCriteriaCert
 import sec_certs.helpers as helpers
 
@@ -17,20 +16,21 @@ class TestCommonCriteriaOOP(TestCase):
     def setUp(self):
         self.test_data_dir = Path(__file__).parent / 'data' / 'test_cc_oop'
         self.crt_one = CommonCriteriaCert('Access Control Devices and Systems',
-                                      'NetIQ Identity Manager 4.7',
-                                      'NetIQ Corporation',
-                                      'SE',
-                                      {'ALC_FLR.2',
-                                       'EAL3+'},
-                                      date(2020, 6, 15),
-                                      date(2025, 6, 15),
-                                      'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf',
-                                      'http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf',
-                                      'csv + html',
-                                      'http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf',
-                                      'https://www.netiq.com/',
-                                      set(),
-                                      set())
+                                          'NetIQ Identity Manager 4.7',
+                                          'NetIQ Corporation',
+                                          'SE',
+                                          {'ALC_FLR.2',
+                                           'EAL3+'},
+                                          date(2020, 6, 15),
+                                          date(2025, 6, 15),
+                                          'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf',
+                                          'http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf',
+                                          'csv + html',
+                                          'http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf',
+                                          'https://www.netiq.com/',
+                                          set(),
+                                          set(),
+                                          None)
 
         self.crt_two = CommonCriteriaCert('Access Control Devices and Systems',
                                           'Magic SSO V4.0',
@@ -46,7 +46,8 @@ class TestCommonCriteriaOOP(TestCase):
                                           'https://www.dreamsecurity.com/',
                                           {CommonCriteriaCert.ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0',
                                                                                 'http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf')},
-                                          set())
+                                          set(),
+                                          None)
 
         pp = CommonCriteriaCert.ProtectionProfile('sample_pp', 'http://sample.pp')
         update = CommonCriteriaCert.MaintainanceReport(date(1900, 1, 1), 'Sample maintainance', 'https://maintainance.up', 'https://maintainance.up')
@@ -63,7 +64,8 @@ class TestCommonCriteriaOOP(TestCase):
                                                  'http://path.to/cert/link',
                                                  'http://path.to/manufacturer/web',
                                                  {pp},
-                                                 {update})
+                                                 {update},
+                                                 None)
         self.template_dataset = CCDataset({self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, Path('/fictional/path/to/dataset'), 'toy dataset', 'toy dataset description')
         self.template_dataset.timestamp = datetime(2020, 11, 16, hour=17, minute=4, second=14, microsecond=770153)
 
@@ -81,27 +83,29 @@ class TestCommonCriteriaOOP(TestCase):
                          'Report link contains some improperly escaped characters.')
 
     def test_download_and_convert_pdfs(self):
-        with open(self.test_data_dir / 'toy_dataset.json', 'r') as handle:
-            dset = json.load(handle, cls=CustomJSONDecoder)
+        dset = CCDataset.from_json(self.test_data_dir / 'toy_dataset.json')
 
         with TemporaryDirectory() as td:
             dset.root_dir = Path(td)
-
             dset.download_all_pdfs()
             dset.convert_all_pdfs()
 
-            actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.report_pdf_paths.items()}
-            actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val) for key, val in dset.target_pdf_paths.items()}
+            actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.report_pdf_path) for key, val in dset.certs.items()}
+            actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.st_pdf_path) for key, val in dset.certs.items()}
 
             self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (certificate report) do not the template')
             self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template')
 
-            self.assertTrue(dset.report_txt_paths['869415cc4b91282e'].exists())
-            self.assertTrue(dset.target_txt_paths['869415cc4b91282e'].exists())
-            self.assertAlmostEqual(dset.target_txt_paths['869415cc4b91282e'].stat().st_size,
-                                   self.template_target_txt_path.stat().st_size, delta=1000)
-            self.assertAlmostEqual(dset.report_txt_paths['869415cc4b91282e'].stat().st_size,
-                                   self.template_report_txt_path.stat().st_size, delta=1000)
+            self.assertTrue(dset['869415cc4b91282e'].state.report_txt_path.exists())
+            self.assertTrue(dset['869415cc4b91282e'].state.st_txt_path.exists())
+
+            self.assertAlmostEqual(dset['869415cc4b91282e'].state.st_txt_path.stat().st_size,
+                                   self.template_target_txt_path.stat().st_size,
+                                   delta=1000)
+
+            self.assertAlmostEqual(dset['869415cc4b91282e'].state.report_txt_path.stat().st_size,
+                                   self.template_report_txt_path.stat().st_size,
+                                   delta=1000)
 
     def test_cert_to_json(self):
         with NamedTemporaryFile('w') as tmp: