Improve (de)serialization of objects from/in json

- Every complex serialiazable object now inherits from `ComplexSerializableType`
- All such objects now implement `to_dict()`, `from_dict()` methods
- Some objects for which standalone json file makes sense implement `to_json()`, `from_json()`
- Rewrite test and demos
- Datasets `root_dir` is now not part of serialization, but only internal representation

7 files changed, 74 insertions, 57 deletions

diff --git a/cc_oop_demo.py b/cc_oop_demo.py
index 54e07307..c78759df 100644
--- a/cc_oop_demo.py
+++ b/cc_oop_demo.py
@@ -26,13 +26,10 @@ def main():
     logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.')
 
     # Dump dataset into JSON
-    with open('./debug_dataset/cc_full_dataset.json', 'w') as handle:
-        json.dump(dset, handle, cls=CustomJSONEncoder, indent=4)
+    dset.to_json('./debug_dataset/cc_full_dataset.json')
 
     # Load dataset from JSON
-    with open('./debug_dataset/cc_full_dataset.json', 'r') as handle:
-        new_dset = json.load(handle, cls=CustomJSONDecoder)
-    new_dset.root_dir = Path('/Users/adam/phd/projects/certificates/sec-certs/debug_dataset')
+    new_dset = CCDataset.from_json('./debug_dataset/cc_full_dataset.json')
 
     assert dset == new_dset
 
diff --git a/fips_oop_demo.py b/fips_oop_demo.py
index 6f556394..9e283670 100644
--- a/fips_oop_demo.py
+++ b/fips_oop_demo.py
@@ -16,7 +16,7 @@ def main():
     logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.')
     # Dump dataset into JSON
 
-    dset.dump_to_json()
+    dset.to_json(dset.root_dir / 'fips_full_dataset.json')
     logging.info(f'Dataset saved to {dset.root_dir}/fips_full_dataset.json')
 
     logging.info("Extracting keywords now.")
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index d2585188..724a60e0 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -5,12 +5,14 @@ import logging
 from pathlib import Path
 import os
 import copy
+import json
 
 from abc import ABC, abstractmethod
 from bs4 import Tag, BeautifulSoup, NavigableString
 from typing import Union, Optional, List, Dict, ClassVar, TypeVar, Type
 
 from sec_certs import helpers, extract_certificates
+from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
 
 logger = logging.getLogger(__name__)
 
@@ -42,8 +44,17 @@ class Certificate(ABC):
     def from_dict(cls: Type[T], dct: dict) -> T:
         return cls(*tuple(dct.values()))
 
+    def to_json(self, output_path: Union[Path, str]):
+        with Path(output_path).open('w') as handle:
+            json.dump(self, handle, indent=4, cls=CustomJSONEncoder)
 
-class FIPSCertificate(Certificate):
+    @classmethod
+    def from_json(cls, input_path: Union[Path, str]):
+        with Path(input_path).open('r') as handle:
+            return json.load(handle, cls=CustomJSONDecoder)
+
+
+class FIPSCertificate(Certificate, ComplexSerializableType):
     FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov'
     FIPS_MODULE_URL: ClassVar[
         str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
@@ -323,12 +334,12 @@ class FIPSCertificate(Certificate):
                                [])
 
 
-class CommonCriteriaCert(Certificate):
+class CommonCriteriaCert(Certificate, ComplexSerializableType):
     cc_url = 'http://www.commoncriteriaportal.org'
     empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/'
 
     @dataclass(eq=True, frozen=True)
-    class MaintainanceReport:
+    class MaintainanceReport(ComplexSerializableType):
         """
         Object for holding maintainance reports.
         """
@@ -357,7 +368,7 @@ class CommonCriteriaCert(Certificate):
             return self.maintainance_date < other.maintainance_date
 
     @dataclass(eq=True, frozen=True)
-    class ProtectionProfile:
+    class ProtectionProfile(ComplexSerializableType):
         """
         Object for holding protection profiles.
         """
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 0bfc7fb0..85b95b7c 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -3,7 +3,7 @@ import re
 from datetime import datetime
 import locale
 import logging
-from typing import Dict, List, ClassVar, Collection, TypeVar, Type
+from typing import Dict, List, ClassVar, Collection, TypeVar, Type, Union
 import json
 from importlib import import_module
 
@@ -29,6 +29,7 @@ from sec_certs.constants import FIPS_NOT_AVAILABLE_CERT_SIZE
 import sec_certs.constants as constants
 import sec_certs.download as download
 import sec_certs.cert_processing as cert_processing
+from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
 
 logger = logging.getLogger(__name__)
 
@@ -63,17 +64,28 @@ class Dataset(ABC):
         return str(type(self).__name__) + ':' + self.name + ', ' + str(len(self)) + ' certificates'
 
     def to_dict(self):
-        return {'root_dir': copy.deepcopy(self.root_dir), 'timestamp': self.timestamp,
-                'sha256_digest': self.sha256_digest, 'name': self.name, 'description': self.description,
+        return {'timestamp': self.timestamp, 'sha256_digest': self.sha256_digest,
+                'name': self.name, 'description': self.description,
                 'n_certs': len(self), 'certs': list(self.certs.values())}
 
     @classmethod
     def from_dict(cls, dct: Dict):
         certs = {x.dgst: x for x in dct['certs']}
-        dset = cls(certs, Path(dct['root_dir']), dct['name'], dct['description'])
+        dset = cls(certs, Path('./'), dct['name'], dct['description'])
         assert len(dset) == dct['n_certs']
         return dset
 
+    def to_json(self, output_path: Union[str, Path]):
+        with Path(output_path).open('w') as handle:
+            json.dump(self, handle, indent=4, cls=CustomJSONEncoder)
+
+    @classmethod
+    def from_json(cls, input_path: Union[str, Path]):
+        with Path(input_path).open('r') as handle:
+            dset = json.load(handle, cls=CustomJSONDecoder)
+        dset.root_path = input_path.parent
+        return dset
+
     @abstractmethod
     def get_certs_from_web(self):
         raise NotImplementedError('Not meant to be implemented by the base class.')
@@ -122,7 +134,7 @@ class Dataset(ABC):
                     # TODO: Delete
 
 
-class CCDataset(Dataset):
+class CCDataset(Dataset, ComplexSerializableType):
     def __init__(self, certs: Dict[str, 'CommonCriteriaCert'], root_dir: Path, name: str = 'dataset name',
                  description: str = 'dataset_description'):
         super().__init__(certs, root_dir, name, description)
@@ -442,7 +454,7 @@ class CCDataset(Dataset):
         self._convert_targets_to_txt()
 
 
-class FIPSDataset(Dataset):
+class FIPSDataset(Dataset, ComplexSerializableType):
     FIPS_BASE_URL: ClassVar[str] = 'https://csrc.nist.gov'
     FIPS_MODULE_URL: ClassVar[
         str] = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
@@ -490,11 +502,6 @@ class FIPSDataset(Dataset):
             self.keywords = json.loads(
                 open(self.root_dir / 'fips_full_keywords.json').read())
 
-    def dump_to_json(self):
-        with open(self.root_dir / 'fips_full_dataset.json', 'w') as handle:
-            json.dump(self, handle, cls=import_module(
-                'sec_certs.serialization').CustomJSONEncoder, indent=4)
-
     def dump_keywords(self):
         with open(self.root_dir / "fips_full_keywords.json", 'w') as f:
             f.write(json.dumps(self.keywords, indent=4, sort_keys=True))
diff --git a/sec_certs/serialization.py b/sec_certs/serialization.py
index 15c38a7b..d37dd067 100644
--- a/sec_certs/serialization.py
+++ b/sec_certs/serialization.py
@@ -2,18 +2,24 @@ import json
 from datetime import date
 from pathlib import Path
 
-from sec_certs.dataset import CCDataset, FIPSDataset
-from sec_certs.certificate import CommonCriteriaCert, FIPSCertificate
+from abc import ABC, abstractmethod
 
-serializable_complex_types = (
-CCDataset, FIPSDataset, CommonCriteriaCert, CommonCriteriaCert.MaintainanceReport, CommonCriteriaCert.ProtectionProfile,
-FIPSCertificate)
-serializable_complex_types_dict = {x.__name__: x for x in serializable_complex_types}
+
+class ComplexSerializableType(ABC):
+    @classmethod
+    @abstractmethod
+    def to_dict(cls):
+        raise NotImplementedError
+
+    @classmethod
+    @abstractmethod
+    def from_dict(cls):
+        raise NotImplementedError
 
 
 class CustomJSONEncoder(json.JSONEncoder):
     def default(self, obj):
-        if isinstance(obj, serializable_complex_types):
+        if isinstance(obj, ComplexSerializableType):
             return {**{'_type': type(obj).__name__}, **obj.to_dict()}
         if isinstance(obj, set):
             return sorted(list(obj))
@@ -25,12 +31,18 @@ class CustomJSONEncoder(json.JSONEncoder):
 
 
 class CustomJSONDecoder(json.JSONDecoder):
+    """
+    Custom JSONDecoder. Any complex object that should be de-serializable must inherit directly from class
+    ComplexSerializableType (nested inheritance does not currently work (because x.__subclassess__() prints only direct
+    subclasses. Any such class must implement methods to_dict() and from_dict(). These are used to drive serialization.
+    """
     def __init__(self, *args, **kwargs):
         json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
+        self.serializable_complex_types = {x.__name__: x for x in ComplexSerializableType.__subclasses__()}
 
     def object_hook(self, obj):
-        if '_type' in obj and obj['_type'] in serializable_complex_types_dict.keys():
+        if '_type' in obj and obj['_type'] in self.serializable_complex_types.keys():
             complex_type = obj.pop('_type')
-            return serializable_complex_types_dict[complex_type].from_dict(obj)
+            return self.serializable_complex_types[complex_type].from_dict(obj)
 
         return obj
diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json
index 24392ad8..a16be012 100644
--- a/test/data/test_cc_oop/toy_dataset.json
+++ b/test/data/test_cc_oop/toy_dataset.json
@@ -1,6 +1,5 @@
 {
     "_type": "CCDataset",
-    "root_dir": "/fictional/path/to/dataset",
     "timestamp": "2020-11-16 17:04:14.770153",
     "sha256_digest": "not implemented",
     "name": "toy dataset",
diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py
index 4ad9b320..98c11a4e 100644
--- a/test/test_cc_oop.py
+++ b/test/test_cc_oop.py
@@ -1,6 +1,6 @@
 from unittest import TestCase
 from pathlib import Path
-from tempfile import TemporaryDirectory, mkstemp
+from tempfile import TemporaryDirectory, mkstemp, NamedTemporaryFile
 from datetime import date, datetime
 import json
 import filecmp
@@ -80,23 +80,6 @@ class TestCommonCriteriaOOP(TestCase):
                          'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf',
                          'Report link contains some improperly escaped characters.')
 
-    @staticmethod
-    def equal_to_json(referential_path, obj):
-        fd, path = mkstemp()
-        try:
-            with os.fdopen(fd, 'w') as handle:
-                json.dump(obj, handle, cls=CustomJSONEncoder, indent=4)
-
-            return filecmp.cmp(referential_path, path)
-        finally:
-            os.remove(path)
-
-    @staticmethod
-    def equal_from_json(referential_path, obj):
-        with open(referential_path, 'r') as handle:
-            new_obj = json.load(handle, cls=CustomJSONDecoder)
-        return obj == new_obj
-
     def test_download_and_convert_pdfs(self):
         with open(self.test_data_dir / 'toy_dataset.json', 'r') as handle:
             dset = json.load(handle, cls=CustomJSONDecoder)
@@ -121,20 +104,28 @@ class TestCommonCriteriaOOP(TestCase):
                                    self.template_report_txt_path.stat().st_size, delta=1000)
 
     def test_cert_to_json(self):
-        self.assertTrue(self.equal_to_json(self.test_data_dir / 'fictional_cert.json', self.fictional_cert),
-                        'The certificate serialized to json differs from a template.')
+        with NamedTemporaryFile('w') as tmp:
+            self.fictional_cert.to_json(tmp.name)
+            self.assertTrue(filecmp.cmp(self.test_data_dir / 'fictional_cert.json',
+                                        tmp.name),
+                            'The certificate serialized to json differs from a template.')
 
     def test_dataset_to_json(self):
-        self.assertTrue(self.equal_to_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset),
-                        'The dataset serialized to json differs from a template.')
+        with NamedTemporaryFile('w') as tmp:
+            self.template_dataset.to_json(tmp.name)
+            self.assertTrue(filecmp.cmp(self.test_data_dir / 'toy_dataset.json',
+                                        tmp.name),
+                            'The dataset serialized to json differs from a template.')
 
     def test_cert_from_json(self):
-        self.assertTrue(self.equal_from_json(self.test_data_dir / 'fictional_cert.json',  self.fictional_cert),
-                        'The certificate serialized from json differs from a template.')
+        self.assertEqual(self.fictional_cert,
+                         CommonCriteriaCert.from_json(self.test_data_dir / 'fictional_cert.json'),
+                         'The certificate serialized from json differs from a template.')
 
     def test_dataset_from_json(self):
-        self.assertTrue(self.equal_from_json(self.test_data_dir / 'toy_dataset.json', self.template_dataset),
-                        'The dataset serialized from json differs from a template.')
+        self.assertEqual(self.template_dataset,
+                         CCDataset.from_json(self.test_data_dir / 'toy_dataset.json'),
+                         'The dataset serialized from json differs from a template.')
 
     def test_build_empty_dataset(self):
         with TemporaryDirectory() as tmp_dir: