WiP new cve and cpe dataset handling

11 files changed, 765 insertions, 335 deletions

diff --git a/src/sec_certs/configuration.py b/src/sec_certs/configuration.py
index 27db673c..44523c7f 100644
--- a/src/sec_certs/configuration.py
+++ b/src/sec_certs/configuration.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import Optional
+from typing import Literal, Optional
 
 import yaml
 from pydantic import AnyHttpUrl, BaseSettings, Field
@@ -69,6 +69,16 @@ class Configuration(BaseSettings):
     fips_mip_latest_snapshot: AnyHttpUrl = Field(
         "https://seccerts.org/fips/mip/latest.json", description="URL for the latest snapshot of FIPS MIP data"
     )
+    cpe_latest_snapshot: AnyHttpUrl = Field(
+        "https://seccerts.org/cpe/cpe_dataset.json.gz", description="URL for the latest snapshot of CPEDataset."
+    )
+    cve_latest_snapshot: AnyHttpUrl = Field(
+        "https://seccerts.org/cve/cve_dataset.json.gz", description="URL for the latest snapshot of CVEDataset."
+    )
+    cpe_match_latest_snapshot: AnyHttpUrl = Field(
+        "https://seccerts.org/cpe/cpe_match_dataset.json.gz",
+        description="URL for the latest snapshot of cpe match json.",
+    )
     minimal_token_length: int = Field(
         3,
         description="Minimal length of a string that will be considered as a token during keyword extraction in CVE matching",
@@ -88,7 +98,13 @@ class Configuration(BaseSettings):
     enable_progress_bars: bool = Field(
         True, description="If true, progress bars will be printed to stdout during computation."
     )
-    nist_api_key: Optional[str] = Field(None, description="NIST API key for access to CVEs and CPEs.")  # noqa: UP007
+    nvd_api_key: Optional[str] = Field(None, description="NVD API key for access to CVEs and CPEs.")  # noqa: UP007
+    preferred_source_nvd_datasets: Literal["sec-certs", "api"] = Field(
+        "sec-certs",
+        description="If set to `sec-certs`, will fetch CPE and CVE datasets from seccerts.org."
+        + " If set to `api`, will fetch these resources from NVD API. It is advised to set an"
+        + " `nvd_api_key` when setting this to `nvd`.",
+    )
 
     def _get_nondefault_keys(self) -> set[str]:
         """
diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
index 684662d0..56910307 100644
--- a/src/sec_certs/constants.py
+++ b/src/sec_certs/constants.py
@@ -1,12 +1,15 @@
 import re
 from pathlib import Path
+from typing import Final
 
 DUMMY_NONEXISTING_PATH = Path("/this/is/dummy/nonexisting/path")
 
 RESPONSE_OK = 200
 RETURNCODE_OK = "ok"
 RETURNCODE_NOK = "nok"
-REQUEST_TIMEOUT = 10
+REQUEST_TIMEOUT = 20
+
+INCREMENTAL_NVD_UPDATE_MAX_INTERVAL_DAYS: Final[int] = 120
 
 MIN_CORRECT_CERT_SIZE = 5000
 
diff --git a/src/sec_certs/dataset/cpe.py b/src/sec_certs/dataset/cpe.py
index 1a20c71e..83d70945 100644
--- a/src/sec_certs/dataset/cpe.py
+++ b/src/sec_certs/dataset/cpe.py
@@ -1,16 +1,20 @@
 from __future__ import annotations
 
 import copy
+import gzip
 import itertools
 import logging
+import shutil
 import tempfile
 import xml.etree.ElementTree as ET
 import zipfile
+from datetime import datetime
 from pathlib import Path
-from typing import ClassVar, Iterator
+from typing import Any, ClassVar, Iterator
 
 import pandas as pd
 
+import sec_certs.configuration as config_module
 from sec_certs import constants
 from sec_certs.dataset.cve import CVEDataset
 from sec_certs.dataset.json_path_dataset import JSONPathDataset
@@ -27,18 +31,17 @@ class CPEDataset(JSONPathDataset, ComplexSerializableType):
     Dataset of CPE records. Includes look-up dictionaries for fast search.
     """
 
-    CPE_XML_BASENAME: ClassVar[str] = "official-cpe-dictionary_v2.3.xml"
-    CPE_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/xml/cpe/dictionary/" + CPE_XML_BASENAME + ".zip"
-
     def __init__(
         self,
-        was_enhanced_with_vuln_cpes: bool,
-        cpes: dict[str, CPE],
+        was_enhanced_with_vuln_cpes: bool = False,
+        cpes: dict[str, CPE] = {},
         json_path: str | Path = constants.DUMMY_NONEXISTING_PATH,
+        last_update_timestamp: datetime = datetime.fromtimestamp(0),
     ):
         self.was_enhanced_with_vuln_cpes = was_enhanced_with_vuln_cpes
         self.cpes = cpes
         self.json_path = Path(json_path)
+        self.last_update_timestamp = last_update_timestamp
 
         self.vendor_to_versions: dict[str, set[str]] = {}
         self.vendor_version_to_cpe: dict[tuple[str, str], set[CPE]] = {}
@@ -56,6 +59,9 @@ class CPEDataset(JSONPathDataset, ComplexSerializableType):
     def __setitem__(self, key: str, value: CPE) -> None:
         self.cpes.__setitem__(key.lower(), value)
 
+    def __delitem__(self, key: str) -> None:
+        self.cpes.__delitem__[key]
+
     def __len__(self) -> int:
         return len(self.cpes)
 
@@ -69,7 +75,7 @@ class CPEDataset(JSONPathDataset, ComplexSerializableType):
 
     @property
     def serialized_attributes(self) -> list[str]:
-        return ["was_enhanced_with_vuln_cpes", "cpes"]
+        return ["last_update_timestamp", "was_enhanced_with_vuln_cpes", "cpes"]
 
     def build_lookup_dicts(self) -> None:
         """
@@ -94,6 +100,11 @@ class CPEDataset(JSONPathDataset, ComplexSerializableType):
                     self.title_to_cpes[cpe.title].add(cpe)
 
     @classmethod
+    def from_dict(cls, dct: dict[str, Any]) -> CPEDataset:
+        dct["last_update_timestamp"] = datetime.fromisoformat(dct["last_update_timestamp"])
+        return cls(**dct)
+
+    @classmethod
     def from_web(cls, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> CPEDataset:
         """
         Creates CPEDataset from NIST resources published on-line
@@ -102,38 +113,35 @@ class CPEDataset(JSONPathDataset, ComplexSerializableType):
         :return CPEDataset: The resulting dataset
         """
         with tempfile.TemporaryDirectory() as tmp_dir:
-            xml_path = Path(tmp_dir) / cls.CPE_XML_BASENAME
-            zip_path = Path(tmp_dir) / (cls.CPE_XML_BASENAME + ".zip")
-            helpers.download_file(cls.CPE_URL, zip_path)
+            dset_path = Path(tmp_dir) / "cpe_dataset.json.gz"
+            helpers.download_file(
+                config_module.config.cpe_latest_snapshot, dset_path, progress_bar_desc="Downloading CPEDataset from web"
+            )
+            dset = cls.from_json(dset_path, is_compressed=True)
 
-            with zipfile.ZipFile(zip_path, "r") as zip_ref:
-                zip_ref.extractall(tmp_dir)
+        dset.json_path = json_path
+        dset.to_json()
+        return dset
 
-            return cls._from_xml(xml_path, json_path)
+    def enhance_with_nvd_data(self, nvd_data: dict[Any, Any]) -> None:
+        self.last_update_timestamp = datetime.fromisoformat(nvd_data["timestamp"])
+        cpes_to_deprecate: set[str] = set()
 
-    @classmethod
-    def _from_xml(cls, xml_path: str | Path, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> CPEDataset:
-        logger.info("Loading CPE dataset from XML.")
-        root = ET.parse(xml_path).getroot()
-        dct = {}
-        for cpe_item in root.findall("{http://cpe.mitre.org/dictionary/2.0}cpe-item"):
-            found_title = cpe_item.find("{http://cpe.mitre.org/dictionary/2.0}title")
-            if found_title is None:
-                raise RuntimeError(
-                    "Title is not found during building CPE dataset from xml - this should not be happening"
-                )
-            title = found_title.text
+        for cpe in nvd_data["products"]:
+            if cpe["cpe"]["deprecated"]:
+                cpes_to_deprecate.add(cpe["cpe"]["cpeNameId"])
+            else:
+                new_cpe = CPE.from_nvd_dict(cpe["cpe"])
+                self.cpes[new_cpe.uri] = new_cpe
 
-            found_cpe_uri = cpe_item.find("{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item")
-            if found_cpe_uri is None:
-                raise RuntimeError(
-                    "CPE uri is not found during building CPE dataset from xml - this should not be happening"
-                )
-            cpe_uri = found_cpe_uri.attrib["name"]
+        uris_to_delete = self._find_uris_for_ids(cpes_to_deprecate)
+        for uri in uris_to_delete:
+            del self[uri]
 
-            dct[cpe_uri] = cached_cpe(cpe_uri, title)
+        self.build_lookup_dicts()
 
-        return cls(False, dct, json_path)
+    def _find_uris_for_ids(self, ids: set[str]) -> set[str]:
+        return {x.uri for x in self if x.uri in ids}
 
     def to_pandas(self) -> pd.DataFrame:
         """
diff --git a/src/sec_certs/dataset/cve.py b/src/sec_certs/dataset/cve.py
index 7217e1c9..8f73405c 100644
--- a/src/sec_certs/dataset/cve.py
+++ b/src/sec_certs/dataset/cve.py
@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import collections
-import datetime
 import glob
 import itertools
 import json
@@ -9,12 +8,14 @@ import logging
 import shutil
 import tempfile
 import zipfile
+from datetime import datetime
 from pathlib import Path
-from typing import ClassVar
+from typing import Any, ClassVar
 
 import numpy as np
 import pandas as pd
 
+import sec_certs.configuration as config_module
 from sec_certs import constants
 from sec_certs.dataset.json_path_dataset import JSONPathDataset
 from sec_certs.sample.cpe import CPE, cached_cpe
@@ -31,11 +32,17 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
     CVE_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-"
     CPE_MATCH_FEED_URL: ClassVar[str] = "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip"
 
-    def __init__(self, cves: dict[str, CVE], json_path: str | Path = constants.DUMMY_NONEXISTING_PATH):
+    def __init__(
+        self,
+        cves: dict[str, CVE] = {},
+        json_path: str | Path = constants.DUMMY_NONEXISTING_PATH,
+        last_update_timestamp: datetime = datetime.fromtimestamp(0),
+    ):
         self.cves = cves
         self.json_path = Path(json_path)
         self.cpe_to_cve_ids_lookup: dict[str, set[str]] = {}
         self.cves_with_vulnerable_configurations: list[CVE] = []
+        self.last_update_timestamp = last_update_timestamp
 
     @property
     def serialized_attributes(self) -> list[str]:
@@ -56,7 +63,26 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
     def __eq__(self, other: object):
         return isinstance(other, CVEDataset) and self.cves == other.cves
 
-    def _filter_cves_with_cpe_configurations(self) -> None:
+    @classmethod
+    def from_web(cls, json_path: str | Path = constants.DUMMY_NONEXISTING_PATH) -> CVEDataset:
+        """
+        Creates CVEDataset from NIST resources published on-line
+
+        :param Union[str, Path] json_path: Path to store the dataset to
+        :return CVEDataset: The resulting dataset
+        """
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            dset_path = Path(tmp_dir) / "cve_dataset.json.gz"
+            helpers.download_file(
+                config_module.config.cve_latest_snapshot, dset_path, progress_bar_desc="Downloading CVEDataset from web"
+            )
+            dset = cls.from_json(dset_path, is_compressed=True)
+
+        dset.json_path = json_path
+        dset.to_json()
+        return dset
+
+    def _get_cves_with_criteria_configurations(self) -> None:
         """
         Method filters the subset of CVE dataset thah contain at least one CPE configuration in the CVE.
         """
@@ -64,105 +90,37 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
 
     def build_lookup_dict(
         self,
-        use_nist_mapping: bool = True,
-        nist_matching_filepath: Path | None = None,
-        limit_to_cpes: set[CPE] | None = None,
+        cpe_match_dict: dict,
+        limit_to_cpes: set[CPE],
     ):
         """
         Builds look-up dictionary CPE -> Set[CVE] and filter the CVEs which contain CPE configurations.
-        Developer's note: There are 3 CPEs that are present in the cpe matching feed, but are badly processed by CVE
-        feed, in which case they won't be found as a key in the dictionary. We intentionally ignore those. Feel free
-        to add corner cases and manual fixes. According to our investigation, the suffereing CPEs are:
-            - CPE(uri='cpe:2.3:a:arubanetworks:airwave:*:*:*:*:*:*:*:*', title=None, version='*', vendor='arubanetworks', item_name='airwave', start_version=None, end_version=('excluding', '8.2.0.0'))
-            - CPE(uri='cpe:2.3:a:bayashi:dopvcomet\\*:0009:b:*:*:*:*:*:*', title=None, version='0009', vendor='bayashi', item_name='dopvcomet\\*', start_version=None, end_version=None)
-            - CPE(uri='cpe:2.3:a:bayashi:dopvstar\\*:0091:*:*:*:*:*:*:*', title=None, version='0091', vendor='bayashi', item_name='dopvstar\\*', start_version=None, end_version=None)
         """
-        self.cpe_to_cve_ids_lookup = {}
-        self.cves = {x.cve_id.upper(): x for x in self}
-
-        logger.info("Getting CPE matching dictionary from NIST.gov")
-
-        if use_nist_mapping:
-            matching_dict = self.get_nist_cpe_matching_dict(nist_matching_filepath)
+        self.cpe_to_cve_ids_lookup = dict.fromkeys([x.uri for x in limit_to_cpes], set())
 
         cve: CVE
         for cve in tqdm(self, desc="Building-up lookup dictionaries for fast CVE matching"):
-            # Filter to CVEs that contain relevant CPEs
-            if limit_to_cpes and not (
-                set(cve.vulnerable_cpes).union(
-                    set(itertools.chain.from_iterable(x.get_all_cpes() for x in cve.vulnerable_cpe_configurations))
-                )
-            ).intersection(limit_to_cpes):
+            vulnerable_cpes = set(
+                itertools.chain.from_iterable([cpe_match_dict[x]["matches"] for x in cve.vulnerable_criteria])
+            )
+            if not cve.vulnerable_criteria_configurations and not any(x in limit_to_cpes for x in vulnerable_cpes):
                 continue
 
-            # See note above, we use matching_dict.get(cpe, []) instead of matching_dict[cpe] as would be expected
-            if use_nist_mapping:
-                vulnerable_configurations = list(
-                    itertools.chain.from_iterable(matching_dict.get(cpe, []) for cpe in cve.vulnerable_cpes)
-                )
-            else:
-                vulnerable_configurations = cve.vulnerable_cpes
-            for cpe in vulnerable_configurations:
-                if cpe.uri not in self.cpe_to_cve_ids_lookup:
-                    self.cpe_to_cve_ids_lookup[cpe.uri] = {cve.cve_id}
-                else:
-                    self.cpe_to_cve_ids_lookup[cpe.uri].add(cve.cve_id)
-
-        self._filter_cves_with_cpe_configurations()
-
-    @classmethod
-    def download_cves(cls, output_path_str: str, start_year: int, end_year: int):
-        output_path = Path(output_path_str)
-        if not output_path.exists():
-            output_path.mkdir()
-
-        urls = [cls.CVE_URL + str(x) + ".json.zip" for x in range(start_year, end_year + 1)]
-
-        logger.info(f"Identified {len(urls)} CVE files to fetch from nist.gov. Downloading them into {output_path}")
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            outpaths = [Path(tmp_dir) / Path(x).name.rstrip(".zip") for x in urls]
-            responses = helpers.download_parallel(urls, outpaths, "Downloading CVEs resources from NVD")
+            for cpe in vulnerable_cpes:
+                self.cpe_to_cve_ids_lookup[cpe.uri].add(cve.cve_id)
 
-            for o, r in zip(outpaths, responses):
-                if r == constants.RESPONSE_OK:
-                    with zipfile.ZipFile(o, "r") as zip_handle:
-                        zip_handle.extractall(output_path)
-
-    @classmethod
-    def from_nist_json(cls, input_path: str) -> CVEDataset:
-        with Path(input_path).open("r") as handle:
-            data = json.load(handle)
-        cves = [CVE.from_nist_dict(x) for x in data["CVE_Items"]]
-        return cls({x.cve_id: x for x in cves})
-
-    @classmethod
-    def from_web(
-        cls,
-        start_year: int = 2002,
-        end_year: int = datetime.datetime.now().year,
-        json_path: str | Path = constants.DUMMY_NONEXISTING_PATH,
-    ):
-        logger.info("Building CVE dataset from nist.gov website.")
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            cls.download_cves(tmp_dir, start_year, end_year)
-            json_files = glob.glob(tmp_dir + "/*.json")
-
-            logger.info("Downloaded required resources. Building CVEDataset from jsons.")
-            results = process_parallel(
-                cls.from_nist_json,
-                json_files,
-                use_threading=False,
-                progress_bar_desc="Building CVEDataset from jsons",
-            )
-        return cls(dict(collections.ChainMap(*(x.cves for x in results))), json_path)
+        self._get_cves_with_criteria_configurations()
 
     def _get_cve_ids_for_cpe_uri(self, cpe_uri: str) -> set[str]:
+        # TODO: Refactor me
         return self.cpe_to_cve_ids_lookup.get(cpe_uri, set())
 
     def _get_cves_from_exactly_matched_cpes(self, cpe_uris: set[str]) -> set[str]:
+        # TODO: Refactor me
         return set(itertools.chain.from_iterable([self._get_cve_ids_for_cpe_uri(cpe_uri) for cpe_uri in cpe_uris]))
 
     def _get_cves_from_cpe_configurations(self, cpe_uris: set[str]) -> set[str]:
+        # TODO: refactor me
         return {
             cve.cve_id
             for cve in self.cves_with_vulnerable_configurations
@@ -170,6 +128,7 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
         }
 
     def get_cves_from_matched_cpes(self, cpe_uris: set[str]) -> set[str]:
+        # TODO: refactor me
         """
         Method returns the set of CVEs which are matched to the set of CPEs.
         First are matched the classic CPEs to CVEs with lookup dict and then are matched the
@@ -181,6 +140,7 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
         }
 
     def filter_related_cpes(self, relevant_cpes: set[CPE]):
+        # TODO: Refactor me
         """
         Since each of the CVEs is related to many CPEs, the dataset size explodes (serialized). For certificates,
         only CPEs within sample dataset are relevant. This function modifies all CVE elements. Specifically, it
@@ -213,53 +173,10 @@ class CVEDataset(JSONPathDataset, ComplexSerializableType):
         df.cwe_ids = df.cwe_ids.map(lambda x: x if x else np.nan)
         return df.set_index("cve_id")
 
-    def get_nist_cpe_matching_dict(self, input_filepath: Path | None) -> dict[CPE, list[CPE]]:
-        """
-        Computes dictionary that maps complex CPEs to list of simple CPEs.
-        """
-
-        def parse_key_cpe(field: dict) -> CPE:
-            start_version = None
-            if "versionStartIncluding" in field:
-                start_version = ("including", field["versionStartIncluding"])
-            elif "versionStartExcluding" in field:
-                start_version = ("excluding", field["versionStartExcluding"])
-
-            end_version = None
-            if "versionEndIncluding" in field:
-                end_version = ("including", field["versionEndIncluding"])
-            elif "versionEndExcluding" in field:
-                end_version = ("excluding", field["versionEndExcluding"])
-
-            return cached_cpe(field["cpe23Uri"], start_version=start_version, end_version=end_version)
-
-        def parse_values_cpe(field: dict) -> list[CPE]:
-            return [cached_cpe(x["cpe23Uri"]) for x in field["cpe_name"]]
-
-        logger.debug("Attempting to get NIST mapping file.")
-        if not input_filepath or not input_filepath.is_file():
-            logger.debug("NIST mapping file not available, going to download.")
-            with tempfile.TemporaryDirectory() as tmp_dir:
-                filename = Path(self.CPE_MATCH_FEED_URL).name
-                download_path = Path(tmp_dir) / filename
-                unzipped_path = Path(tmp_dir) / filename.rstrip(".zip")
-                helpers.download_file(self.CPE_MATCH_FEED_URL, download_path)
-
-                with zipfile.ZipFile(download_path, "r") as zip_handle:
-                    zip_handle.extractall(tmp_dir)
-                with unzipped_path.open("r") as handle:
-                    match_data = json.load(handle)
-                if input_filepath:
-                    logger.debug(f"Copying attained NIST mapping file to {input_filepath}")
-                    shutil.move(str(unzipped_path), str(input_filepath))
-        else:
-            with input_filepath.open("r") as handle:
-                match_data = json.load(handle)
-
-        mapping_dict = {}
-        for match in tqdm(match_data["matches"], desc="parsing cpe matching (by NIST) dictionary"):
-            key = parse_key_cpe(match)
-            value = parse_values_cpe(match)
-            mapping_dict[key] = value if value else [key]
-
-        return mapping_dict
+    def enhance_with_nvd_data(self, data: dict[str, Any]) -> None:
+        self.last_update_timestamp = datetime.fromisoformat(data["timestamp"])
+        for vuln in data["vulnerabilities"]:
+            # https://nvd.nist.gov/vuln/vulnerability-status#divNvdStatus
+            if vuln["cve"]["vulnStatus"] in {"Analyzed", "Modified"}:
+                cve = CVE.from_nist_dict(vuln["cve"])
+                self[cve.cve_id] = cve
diff --git a/src/sec_certs/dataset/dataset.py b/src/sec_certs/dataset/dataset.py
index d388db2e..2d66c9a0 100644
--- a/src/sec_certs/dataset/dataset.py
+++ b/src/sec_certs/dataset/dataset.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import gzip
 import itertools
 import json
 import logging
@@ -23,6 +24,7 @@ from sec_certs.sample.certificate import Certificate
 from sec_certs.sample.cpe import CPE
 from sec_certs.serialization.json import ComplexSerializableType, get_class_fullname, serialize
 from sec_certs.utils import helpers
+from sec_certs.utils.nvd_dataset_builder import CpeMatchNvdDatasetBuilder, CpeNvdDatasetBuilder, CveNvdDatasetBuilder
 from sec_certs.utils.tqdm import tqdm
 
 logger = logging.getLogger(__name__)
@@ -123,6 +125,10 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         return self.auxiliary_datasets_dir / "cpe_dataset.json"
 
     @property
+    def cpe_match_json_path(self) -> Path:
+        return self.auxiliary_datasets / "cpe_match_feed.json"
+
+    @property
     def cve_dataset_path(self) -> Path:
         return self.auxiliary_datasets_dir / "cve_dataset.json"
 
@@ -195,8 +201,8 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         return dset
 
     @classmethod
-    def from_json(cls: type[DatasetSubType], input_path: str | Path) -> DatasetSubType:
-        dset = cast("DatasetSubType", ComplexSerializableType.from_json(input_path))
+    def from_json(cls: type[DatasetSubType], input_path: str | Path, is_compressed: bool = False) -> DatasetSubType:
+        dset = cast("DatasetSubType", ComplexSerializableType.from_json(input_path, is_compressed))
         dset._root_dir = Path(input_path).parent.absolute()
         dset._set_local_paths()
         return dset
@@ -253,9 +259,11 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
         logger.info("Processing auxiliary datasets.")
         self.auxiliary_datasets_dir.mkdir(parents=True, exist_ok=True)
         self.auxiliary_datasets.cpe_dset = self._prepare_cpe_dataset(download_fresh)
-        self.auxiliary_datasets.cve_dset = self._prepare_cve_dataset(
-            download_fresh_cves=download_fresh, build_lookup_dict=False
-        )
+        self.auxiliary_datasets.cve_dset = self._prepare_cve_dataset(download_fresh)
+
+        if download_fresh or not self.cpe_match_json_path.exists():
+            self._prepare_cpe_match_dict(download_fresh=True)
+
         self.state.auxiliary_datasets_processed = True
 
     @serialize
@@ -334,18 +342,6 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
             self.auxiliary_datasets.cpe_dset = self._prepare_cpe_dataset()
 
         self.compute_cpe_heuristics()
-
-        cpe_rich = [
-            set(map(self.auxiliary_datasets.cpe_dset.cpes.get, x.heuristics.cpe_matches))
-            for x in self
-            if x.heuristics.cpe_matches is not None
-        ]
-        all_cpes = set(itertools.chain.from_iterable(cpe_rich))
-
-        if not self.auxiliary_datasets.cve_dset:
-            self.auxiliary_datasets.cve_dset = self._prepare_cve_dataset(build_lookup_dict=False)
-        self.auxiliary_datasets.cve_dset.build_lookup_dict(use_nist_mapping=True, limit_to_cpes=all_cpes)  # type: ignore
-
         self.compute_related_cves()
         self._compute_references()
         self._compute_transitive_vulnerabilities()
@@ -358,36 +354,79 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
     def _compute_transitive_vulnerabilities(self) -> None:
         raise NotImplementedError("Not meant to be implemented by the base class.")
 
-    def _prepare_cpe_dataset(self, download_fresh_cpes: bool = False) -> CPEDataset:
-        logger.info("Preparing CPE dataset.")
+    def _prepare_cpe_dataset(self, download_fresh: bool = False) -> CPEDataset:
         if not self.auxiliary_datasets_dir.exists():
             self.auxiliary_datasets_dir.mkdir(parents=True)
 
-        if not self.cpe_dataset_path.exists() or download_fresh_cpes is True:
-            cpe_dataset = CPEDataset.from_web(self.cpe_dataset_path)
-            cpe_dataset.to_json()
+        if self.cpe_dataset_path.exists():
+            logger.info("Loading CPEDataset from json.")
+            cpe_dataset = CPEDataset.from_json(self.cpe_dataset_path)
         else:
-            cpe_dataset = CPEDataset.from_json(str(self.cpe_dataset_path))
+            cpe_dataset = CPEDataset(json_path=self.cpe_dataset_path)
+            download_fresh = True
+
+        if download_fresh:
+            if config.preferred_source_nvd_datasets == "api":
+                logger.info("Fetching new CPE records from NVD API.")
+                with CpeNvdDatasetBuilder(api_key=config.nvd_api_key) as builder:
+                    cpe_dataset = builder.build_dataset(cpe_dataset)
+            else:
+                logger.info("Preparing CPEDataset from seccerts.org.")
+                cpe_dataset = CPEDataset.from_web(self.cpe_dataset_path)
+            cpe_dataset.to_json()
 
         return cpe_dataset
 
-    def _prepare_cve_dataset(
-        self, download_fresh_cves: bool = False, use_nist_cpe_matching_dict: bool = True, build_lookup_dict: bool = True
-    ) -> CVEDataset:
-        logger.info("Preparing CVE dataset.")
+    def _prepare_cve_dataset(self, download_fresh: bool = False) -> CVEDataset:
         if not self.auxiliary_datasets_dir.exists():
+            logger.info("Loading CVEDataset from json.")
             self.auxiliary_datasets_dir.mkdir(parents=True)
 
-        if not self.cve_dataset_path.exists() or download_fresh_cves is True:
-            cve_dataset = CVEDataset.from_web(json_path=self.cve_dataset_path)
-            cve_dataset.to_json()
-        else:
+        if self.cve_dataset_path.exists():
             cve_dataset = CVEDataset.from_json(self.cve_dataset_path)
+        else:
+            cve_dataset = CVEDataset(json_path=self.cve_dataset_path)
+            download_fresh = True
+
+        if download_fresh:
+            if config.preferred_source_nvd_datasets == "api":
+                logger.info("Fetching new CVE records from NVD API.")
+                with CveNvdDatasetBuilder(api_key=config.nvd_api_key) as builder:
+                    cve_dataset = builder.build_dataset(cve_dataset)
+            else:
+                logger.info("Preparing CVEDataset from seccerts.org")
+                cve_dataset = CVEDataset.from_web(self.cve_dataset_path)
 
-        if build_lookup_dict:
-            cve_dataset.build_lookup_dict(use_nist_cpe_matching_dict, self.nist_cve_cpe_matching_dset_path)
         return cve_dataset
 
+    def _prepare_cpe_match_dict(self, download_fresh: bool = False) -> dict:
+        if self.cpe_match_json_path.exists():
+            with self.cpe_match_json_path.open("r") as handle:
+                cpe_match_dict = json.load(handle)
+        else:
+            cpe_match_dict = CpeMatchNvdDatasetBuilder._init_new_dataset()
+            download_fresh = True
+
+        if download_fresh:
+            if config.preferred_source_nvd_datasets == "api":
+                logger.info("Fetchnig CPE Match feed from NVD APi.")
+                with CpeMatchNvdDatasetBuilder(api_key=config.nvd_api_key) as builder:
+                    cpe_match_dict = builder.build_dataset(cpe_match_dict)
+            else:
+                logger.info("Preparing CPE Match feed from seccerts.org")
+                with tempfile.TemporaryDirectory() as tmp_dir:
+                    dset_path = Path(tmp_dir) / "cpe_match_feed.json.gz"
+                    helpers.download_file(
+                        config.cpe_match_latest_snapshot,
+                        dset_path,
+                        progress_bar_desc="Downloading CPE Match feed from web",
+                    )
+                    with gzip.open(str(dset_path)) as handle:
+                        json_str = handle.read().decode("utf-8")
+                        cpe_match_dict = json.loads(json_str)
+
+        return cpe_match_dict
+
     @serialize
     def compute_cpe_heuristics(self, download_fresh_cpes: bool = False) -> CPEClassifier:
         """
@@ -521,20 +560,29 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
                     set(cert.heuristics.verified_cpe_matches)
                 )
 
+    def _get_all_cpes_in_dataset(self) -> set[CPE]:
+        cpe_rich: list[set[CPE]] = [
+            set(map(self.auxiliary_datasets.cpe_dset.cpes.get, x.heuristics.cpe_matches))
+            for x in self
+            if x.heuristics.cpe_matches
+        ]
+        return set(itertools.chain.from_iterable(cpe_rich))
+
     @serialize
-    def compute_related_cves(
-        self,
-        download_fresh_cves: bool = False,
-        use_nist_cpe_matching_dict: bool = True,
-    ) -> None:
+    def compute_related_cves(self) -> None:
         """
         Computes CVEs for the certificates, given their CPE matches.
         """
         logger.info("Retrieving related CVEs to verified CPE matches")
-        if download_fresh_cves or not self.auxiliary_datasets.cve_dset:
-            self.auxiliary_datasets.cve_dset = self._prepare_cve_dataset(
-                download_fresh_cves, use_nist_cpe_matching_dict
-            )
+        if not self.auxiliary_datasets.cve_dset:
+            self.auxiliary_datasets.cve_dset = self._prepare_cve_dataset()
+
+        if not self.auxiliary_datasets.cpe_dset:
+            self.auxiliary_datasets.cpe_dset = self._prepare_cpe_dataset()
+
+        all_cpes = self._get_all_cpes_in_dataset()
+        cpe_match_dict = self._prepare_cpe_match_dict()
+        self.auxiliary_datasets.cve_dset.build_lookup_dict(cpe_match_dict, all_cpes)
 
         logger.info("Computing heuristics: CVEs in certificates.")
         self.enrich_automated_cpes_with_manual_labels()
diff --git a/src/sec_certs/dataset/json_path_dataset.py b/src/sec_certs/dataset/json_path_dataset.py
index 6eb4d968..b0b0a57b 100644
--- a/src/sec_certs/dataset/json_path_dataset.py
+++ b/src/sec_certs/dataset/json_path_dataset.py
@@ -39,8 +39,7 @@ class JSONPathDataset(ComplexSerializableType, ABC):
             self.to_json()
 
     @classmethod
-    def from_json(cls, input_path: str | Path):
-        with Path(input_path).open("r") as handle:
-            dset = json.load(handle, cls=CustomJSONDecoder)
+    def from_json(cls, input_path: str | Path, is_compressed: bool = False):
+        dset = super().from_json(input_path, is_compressed)
         dset.json_path = Path(input_path)
         return dset
diff --git a/src/sec_certs/sample/cpe.py b/src/sec_certs/sample/cpe.py
index d56535c2..055373c1 100644
--- a/src/sec_certs/sample/cpe.py
+++ b/src/sec_certs/sample/cpe.py
@@ -11,6 +11,59 @@ from sec_certs.utils import helpers
 
 
 @dataclass
+class CPEMatchCriteria(ComplexSerializableType):
+    vulnerable: bool
+    criteria: str
+    criteria_id: str
+    version_start: tuple[str, str] | None
+    version_end: tuple[str, str] | None
+
+    __slots__ = ["vulnerable", "criteria", "criteria_id", "version_start", "version_end"]
+
+    # We cannot use frozen=True. It does not work with __slots__ prior to Python 3.10 dataclasses
+    # Hence we manually provide __hash__ and __eq__ despite not guaranteeing immutability
+    def __hash__(self) -> int:
+        return hash(self.criteria_id)
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, CPEMatchCriteria) and self.criteria_id == other.criteria_id
+
+    def __lt__(self, other: CPEMatchCriteria) -> bool:
+        return self.criteria_id < other.criteria_id
+
+    @classmethod
+    def from_nist_dict(cls, dct: dict[str, Any]) -> CPEMatchCriteria:
+        if dct.get("versionStartIncluding", None):
+            version_start = ("including", dct["versionStartIncluding"])
+        elif dct.get("versionStartExcluding"):
+            version_start = ("excluding", dct["versionStartExcluding"])
+        else:
+            version_start = None
+
+        if dct.get("versionEndIncluding", None):
+            version_end = ("including", dct["versionEndIncluding"])
+        elif dct.get("versionEndExcluding", None):
+            version_end = ("excluding", dct["versionEndExcluding"])
+        else:
+            version_end = None
+
+        return cls(dct["vulnerable"], dct["criteria"], dct["matchCriteriaId"], version_start, version_end)
+
+
+@dataclass
+class CPEMatchCriteriaConfiguration(ComplexSerializableType):
+    """
+    This class represents a set of sets of `CPEMatchCriteria` objects, where there's an OR relation between the
+    elements of the set.
+    Our experiments confirm that there are only 3 distinct CVEs in the database that allow AND configuration between
+    the elements. Simplyfing to ORs enables much more simple implementation.
+    """
+
+    components: list[list[CPEMatchCriteria]]
+    __slots__ = ["components"]
+
+
+@dataclass
 class CPEConfiguration(ComplexSerializableType):
     __slots__ = ["platform", "cpes"]
 
@@ -41,6 +94,7 @@ class CPEConfiguration(ComplexSerializableType):
 
 @dataclass
 class CPE(PandasSerializableType, ComplexSerializableType):
+    cpe_id: str
     uri: str
     version: str
     vendor: str
@@ -49,10 +103,10 @@ class CPE(PandasSerializableType, ComplexSerializableType):
     start_version: tuple[str, str] | None
     end_version: tuple[str, str] | None
 
-    __slots__ = ["uri", "version", "vendor", "item_name", "title", "start_version", "end_version"]
+    __slots__ = ["cpe_id", "uri", "version", "vendor", "item_name", "title", "start_version", "end_version"]
 
     pandas_columns: ClassVar[list[str]] = [
-        "uri",
+        "cpe_id" "uri",
         "vendor",
         "item_name",
         "version",
@@ -61,12 +115,14 @@ class CPE(PandasSerializableType, ComplexSerializableType):
 
     def __init__(
         self,
+        cpe_id: str,
         uri: str,
         title: str | None = None,
         start_version: tuple[str, str] | None = None,
         end_version: tuple[str, str] | None = None,
     ):
         super().__init__()
+        self.cpe_id = cpe_id
         self.uri = uri
 
         splitted = helpers.split_unescape(self.uri, ":")
@@ -97,9 +153,14 @@ class CPE(PandasSerializableType, ComplexSerializableType):
             dct["end_version"] = tuple(dct["end_version"])
         return super().from_dict(dct)
 
+    @classmethod
+    def from_nvd_dict(cls, dct: dict[str, Any]) -> CPE:
+        title = [x for x in dct["titles"] if x["lang"] == "en"][0]["title"]
+        return cls(dct["cpeNameId"], dct["cpeName"], title, None, None)
+
     @property
     def serialized_attributes(self) -> list[str]:
-        return ["uri", "title", "start_version", "end_version"]
+        return ["cpe_id", "uri", "title", "start_version", "end_version"]
 
     @property
     def update(self) -> str:
diff --git a/src/sec_certs/sample/cve.py b/src/sec_certs/sample/cve.py
index 2289b0e1..3e4c3603 100644
--- a/src/sec_certs/sample/cve.py
+++ b/src/sec_certs/sample/cve.py
@@ -1,13 +1,12 @@
 from __future__ import annotations
 
 import datetime
-import itertools
 from dataclasses import dataclass
-from typing import Any, ClassVar, Iterable
+from typing import Any, ClassVar
 
 from dateutil.parser import isoparse
 
-from sec_certs.sample.cpe import CPE, CPEConfiguration, cached_cpe
+from sec_certs.sample.cpe import CPEMatchCriteria, CPEMatchCriteriaConfiguration
 from sec_certs.serialization.json import ComplexSerializableType
 from sec_certs.serialization.pandas import PandasSerializableType
 
@@ -15,7 +14,7 @@ from sec_certs.serialization.pandas import PandasSerializableType
 @dataclass
 class CVE(PandasSerializableType, ComplexSerializableType):
     @dataclass
-    class Impact(ComplexSerializableType):
+    class Metrics(ComplexSerializableType):
         base_score: float
         severity: str
         exploitability_score: float
@@ -24,36 +23,76 @@ class CVE(PandasSerializableType, ComplexSerializableType):
         __slots__ = ["base_score", "severity", "exploitability_score", "impact_score"]
 
         @classmethod
-        def from_nist_dict(cls, dct: dict[str, Any]) -> CVE.Impact:
+        def from_nist_dict(cls, dct: dict[str, Any]) -> CVE.Metrics:
             """
-            Will load Impact from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1
+            Loads metrics from dictionary
             """
-            if not dct["impact"]:
+            if not dct["metrics"]:
                 return cls(0, "", 0, 0)
-            elif "baseMetricV3" in dct["impact"]:
+            metric_dct = CVE.Metrics.find_metrics_to_use(dct["metrics"])
+            if not metric_dct:
+                raise ValueError(f"Metrics dictionary for cve {dct['id']} present, but no suitable entry found.")
+            return CVE.Metrics.from_metrics_dct(metric_dct)
+
+        @staticmethod
+        def find_metrics_to_use(dct: dict) -> dict | None:
+            """
+            any `Primary` entry available > any `nvd@nist.gov` entry available > just return the first entry if exists.
+            """
+            all_metrics = dct.get("cvssMetricV31", []) + dct.get("cvssMetricV30", []) + dct.get("cvssMetricV2", [])
+
+            for element in all_metrics:
+                if element["type"] == "Primary":
+                    return element
+            for element in all_metrics:
+                if element["source"] == "nvd@nist.gov":
+                    return element
+
+            if all_metrics:
+                return all_metrics[0]
+
+            return None
+
+        @classmethod
+        def from_metrics_dct(cls, dct: dict) -> CVE.Metrics:
+            if dct["cvssData"]["version"] == "3.1":
                 return cls(
-                    dct["impact"]["baseMetricV3"]["cvssV3"]["baseScore"],
-                    dct["impact"]["baseMetricV3"]["cvssV3"]["baseSeverity"],
-                    dct["impact"]["baseMetricV3"]["exploitabilityScore"],
-                    dct["impact"]["baseMetricV3"]["impactScore"],
+                    dct["cvssData"]["baseScore"],
+                    dct["cvssData"]["baseSeverity"],
+                    dct["exploitabilityScore"],
+                    dct["impactScore"],
                 )
-            elif "baseMetricV2" in dct["impact"]:
+            if dct["cvssData"]["version"] == "3.0":
                 return cls(
-                    dct["impact"]["baseMetricV2"]["cvssV2"]["baseScore"],
-                    dct["impact"]["baseMetricV2"]["severity"],
-                    dct["impact"]["baseMetricV2"]["exploitabilityScore"],
-                    dct["impact"]["baseMetricV2"]["impactScore"],
+                    dct["cvssData"]["baseScore"],
+                    dct["cvssData"]["baseSeverity"],
+                    dct["exploitabilityScore"],
+                    dct["impactScore"],
                 )
-            raise ValueError("NIST Dict for CVE Impact badly formatted.")
+            if dct["cvssData"]["version"] == "2.0":
+                return cls(
+                    dct["cvssData"]["baseScore"],
+                    dct["baseSeverity"],
+                    dct["exploitabilityScore"],
+                    dct["impactScore"],
+                )
+            raise ValueError(f"Unknown CVSS version occured ({dct['cvssData']['version']}) when parsing CVSS metrics.")
 
     cve_id: str
-    vulnerable_cpes: list[CPE]
-    vulnerable_cpe_configurations: list[CPEConfiguration]
-    impact: Impact
+    vulnerable_criteria: list[CPEMatchCriteria]
+    vulnerable_criteria_configurations: list[CPEMatchCriteriaConfiguration]
+    metrics: Metrics
     published_date: datetime.datetime | None
     cwe_ids: set[str] | None
 
-    __slots__ = ["cve_id", "vulnerable_cpes", "vulnerable_cpe_configurations", "impact", "published_date", "cwe_ids"]
+    __slots__ = [
+        "cve_id",
+        "vulnerable_criteria",
+        "vulnerable_criteria_configurations",
+        "metrics",
+        "published_date",
+        "cwe_ids",
+    ]
 
     pandas_columns: ClassVar[list[str]] = [
         "cve_id",
@@ -88,11 +127,11 @@ class CVE(PandasSerializableType, ComplexSerializableType):
     def pandas_tuple(self):
         return (
             self.cve_id,
-            self.vulnerable_cpes,
-            self.impact.base_score,
-            self.impact.severity,
-            self.impact.exploitability_score,
-            self.impact.impact_score,
+            self.vulnerable_criteria,
+            self.metrics.base_score,
+            self.metrics.severity,
+            self.metrics.exploitability_score,
+            self.metrics.impact_score,
             self.published_date,
             self.cwe_ids,
         )
@@ -100,9 +139,9 @@ class CVE(PandasSerializableType, ComplexSerializableType):
     def to_dict(self) -> dict[str, Any]:
         return {
             "cve_id": self.cve_id,
-            "vulnerable_cpes": self.vulnerable_cpes,
-            "vulnerable_cpe_configurations": self.vulnerable_cpe_configurations,
-            "impact": self.impact,
+            "vulnerable_cpes": self.vulnerable_criteria,
+            "vulnerable_criteria_configurations": self.vulnerable_criteria_configurations,
+            "impact": self.metrics,
             "published_date": self.published_date.isoformat() if self.published_date else None,
             "cwe_ids": self.cwe_ids,
         }
@@ -115,7 +154,7 @@ class CVE(PandasSerializableType, ComplexSerializableType):
         return cls(
             dct["cve_id"],
             dct["vulnerable_cpes"],
-            dct["vulnerable_cpe_configurations"],
+            dct["vulnerable_criteria_configurations"],
             dct["impact"],
             date_to_take,
             dct["cwe_ids"],
@@ -123,92 +162,51 @@ class CVE(PandasSerializableType, ComplexSerializableType):
 
     @classmethod
     def from_nist_dict(cls, dct: dict) -> CVE:
-        cve_id = dct["cve"]["CVE_data_meta"]["ID"]
-        impact = cls.Impact.from_nist_dict(dct)
-        published_date = isoparse(dct["publishedDate"])
+        cve_id = dct["id"]
+        metrics = cls.Metrics.from_nist_dict(dct)
+        published_date = datetime.datetime.fromisoformat(dct["published"])
         cwe_ids = cls.parse_cwe_data(dct)
-        cpes, cpe_configurations = CVE.get_cpe_data_from_nodes_list(dct["configurations"]["nodes"])
-
-        return cls(cve_id, cpes, cpe_configurations, impact, published_date, cwe_ids)
-
-    @staticmethod
-    def _parse_nist_cpe_dicts(dictionaries: Iterable[dict[str, Any]]) -> list[CPE]:
-        cpes: list[CPE] = []
-
-        for x in dictionaries:
-            cpe_uri = x["cpe23Uri"]
-            version_start: tuple[str, str] | None
-            version_end: tuple[str, str] | None
-            if "versionStartIncluding" in x and x["versionStartIncluding"]:
-                version_start = ("including", x["versionStartIncluding"])
-            elif "versionStartExcluding" in x and x["versionStartExcluding"]:
-                version_start = ("excluding", x["versionStartExcluding"])
-            else:
-                version_start = None
-
-            if "versionEndIncluding" in x and x["versionEndIncluding"]:
-                version_end = ("including", x["versionEndIncluding"])
-            elif "versionEndExcluding" in x and x["versionEndExcluding"]:
-                version_end = ("excluding", x["versionEndExcluding"])
-            else:
-                version_end = None
-
-            cpes.append(cached_cpe(cpe_uri, start_version=version_start, end_version=version_end))
-
-        return cpes
-
-    @staticmethod
-    def _parse_nist_dict(cpe_list: list[dict[str, Any]], parse_only_vulnerable_cpes: bool) -> list[CPE]:
-        """
-        Method parses list of CPE dicts to the list of CPE objects.
-        The <parse_only_vulnerable_cpes> parameter specifies if we want to
-        parse only vulnerable CPEs or not.
-        """
-        return CVE._parse_nist_cpe_dicts(dct for dct in cpe_list if dct["vulnerable"] or not parse_only_vulnerable_cpes)
+        vulnerable_criteria, vulnerable_criteria_configurations = CVE.parse_configurations(dct)
+        return cls(cve_id, vulnerable_criteria, vulnerable_criteria_configurations, metrics, published_date, cwe_ids)
 
     @staticmethod
     def parse_cwe_data(dct: dict) -> set[str] | None:
-        descriptions = dct["cve"]["problemtype"]["problemtype_data"][0]["description"]
+        if "weaknesses" not in dct:
+            return None
+        assert dct["weaknesses"][0]["type"] == "Primary"
+        descriptions = dct["weaknesses"][0]["description"]
         return {x["value"] for x in descriptions} if descriptions else None
 
     @staticmethod
-    def get_cpe_data_from_nodes_list(lst: list) -> tuple[list[CPE], list[CPEConfiguration]]:
-        or_nodes = [x for x in lst if x["operator"] == "OR"]
-        and_nodes = [x for x in lst if x["operator"] == "AND"]
-        return CVE.get_simple_cpes_from_nodes_list(or_nodes), CVE.get_cpe_configurations_from_node_list(and_nodes)
+    def parse_configurations(
+        dct: dict[str, Any],
+    ) -> tuple[list[CPEMatchCriteria], list[CPEMatchCriteriaConfiguration]]:
+        criteria = []
+        criteria_configurations = []
 
-    @staticmethod
-    def get_simple_cpes_from_nodes_list(lst: list) -> list[CPE]:
-        return list(
-            itertools.chain.from_iterable(
-                CVE._parse_nist_dict(node["cpe_match"], parse_only_vulnerable_cpes=True) for node in lst
-            )
-        )
+        configurations = dct.get("configurations", [])
+        for conf in configurations:
+            new_criteria, new_criteria_configuration = CVE.parse_single_configuration(conf)
+            criteria.extend(new_criteria)
+            if new_criteria_configuration:
+                criteria_configurations.append(new_criteria_configuration)
+        return criteria, criteria_configurations
 
     @staticmethod
-    def get_cpe_configurations_from_node_list(lst: list) -> list[CPEConfiguration]:
-        """
-        Retrieves only running on/with configurations, not the advanced ones.
-        See more at https://nvd.nist.gov/vuln/vulnerability-detail-pages, section `Configurations`
-        """
-        configurations = [CVE.get_cpe_confiugration_from_node(x) for x in lst]
-        return [x for x in configurations if x]
+    def parse_single_configuration(
+        configuration: dict[str, Any]
+    ) -> tuple[list[CPEMatchCriteria], CPEMatchCriteriaConfiguration | None]:
+        if "operator" not in configuration or configuration["operator"] == "OR":
+            assert len(configuration["nodes"]) == 1 and "cpeMatch" in configuration["nodes"][0]
+            return CVE.get_criteria_from_node(configuration["nodes"][0]["cpeMatch"]), None
 
-    @staticmethod
-    def get_cpe_confiugration_from_node(node: dict) -> CPEConfiguration | None:
-        if node["children"]:
-            if len(node["children"]) != 2:
-                return None
+        return [], CVE.get_configuration_criteria_from_nodes(configuration["nodes"])
 
-            # Deep variant should have two children, get CPEs from the first one and declare that product, second is platform
-            cpes = CVE._parse_nist_dict(node["children"][0]["cpe_match"], parse_only_vulnerable_cpes=True)
-            platform = CVE._parse_nist_dict(node["children"][1]["cpe_match"], parse_only_vulnerable_cpes=False)
-            return CPEConfiguration(platform[0], cpes)
-        else:
-            # Shallow variant should have exactly 2 matching CPEs, we declare one a platform, second one the vuln. thing
-            cpes = CVE._parse_nist_dict(node["cpe_match"], parse_only_vulnerable_cpes=True)
-
-            if len(cpes) != 2:
-                return None
+    @staticmethod
+    def get_configuration_criteria_from_nodes(nodes) -> CPEMatchCriteriaConfiguration:
+        assert all("cpeMatch" in x for x in nodes)  # the next layer are matches
+        return CPEMatchCriteriaConfiguration([CVE.get_criteria_from_node(x["cpeMatch"]) for x in nodes])
 
-            return CPEConfiguration(cpes[0], [cpes[1]])
+    @staticmethod
+    def get_criteria_from_node(cpe_matches: list[dict[str, Any]]) -> list[CPEMatchCriteria]:
+        return [CPEMatchCriteria.from_nist_dict(x) for x in cpe_matches]
diff --git a/src/sec_certs/serialization/json.py b/src/sec_certs/serialization/json.py
index 7b523b9b..7f86ab3b 100644
--- a/src/sec_certs/serialization/json.py
+++ b/src/sec_certs/serialization/json.py
@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import copy
+import gzip
 import json
-from datetime import date
+from datetime import date, datetime
 from functools import wraps
 from pathlib import Path
 from typing import Any, Callable, TypeVar
@@ -44,7 +45,12 @@ class ComplexSerializableType:
         except TypeError as e:
             raise TypeError(f"Dict: {dct} on {cls.__mro__}") from e
 
-    def to_json(self, output_path: str | Path | None = None) -> None:
+    def to_json(self, output_path: str | Path | None = None, compress: bool = False) -> None:
+        """
+        Serializes `ComplexSerializableType` instance to json file.
+        :param str | Path | None output_path: path where the file will be stored. If None, `obj.json_path` access is attempted, defaults to None
+        :param bool compress: if True, will be compress with gzip, defaults to False
+        """
         if not output_path and (not hasattr(self, "json_path") or not self.json_path):  # type: ignore
             raise SerializationError(
                 f"The object {self} of type {self.__class__} does not have json_path attribute set but to_json() was called without an argument."
@@ -60,14 +66,30 @@ class ComplexSerializableType:
             raise SerializationError("output path for json cannot be directory.")
 
         # false positive MyPy warning, cannot be None
-        with Path(output_path).open("w") as handle:  # type: ignore
-            json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
+        if compress:
+            with gzip.open(str(output_path), "w") as handle:  # type: ignore
+                json_str = json.dumps(self, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
+                handle.write(json_str.encode("utf-8"))
+        else:
+            with Path(output_path).open("w") as handle:  # type: ignore
+                json.dump(self, handle, indent=4, cls=CustomJSONEncoder, ensure_ascii=False)
 
     @classmethod
-    def from_json(cls: type[T], input_path: str | Path) -> T:
-        input_path = Path(input_path)
-        with input_path.open("r") as handle:
-            return json.load(handle, cls=CustomJSONDecoder)
+    def from_json(cls: type[T], input_path: str | Path, is_compressed: bool = False) -> T:
+        """
+        Will load `ComplexSerializableType` from json.
+        :param str | Path input_path: path to load the file from
+        :param bool is_compressed: if True, will decompress .gz first, defaults to False
+        :return T: the deserialized object
+        """
+        if is_compressed:
+            with gzip.open(str(input_path)) as handle:
+                json_str = handle.read().decode("utf-8")
+                return json.loads(json_str, cls=CustomJSONDecoder)
+        else:
+            input_path = Path(input_path)
+            with input_path.open("r") as handle:
+                return json.load(handle, cls=CustomJSONDecoder)
 
 
 # Decorator for serialization
@@ -113,6 +135,8 @@ class CustomJSONEncoder(json.JSONEncoder):
             return sorted(obj)
         if isinstance(obj, date):
             return str(obj)
+        if isinstance(obj, datetime):
+            return obj.isoformat()
         if isinstance(obj, Path):
             return str(obj)
         return super().default(obj)
diff --git a/src/sec_certs/utils/helpers.py b/src/sec_certs/utils/helpers.py
index 596ecf62..c5aab70a 100644
--- a/src/sec_certs/utils/helpers.py
+++ b/src/sec_certs/utils/helpers.py
@@ -8,13 +8,14 @@ from contextlib import nullcontext
 from datetime import datetime
 from functools import partial
 from pathlib import Path
-from typing import Any, Collection
+from typing import Any, Collection, Literal
 
 import numpy as np
 import pkgconfig
 import requests
 
 from sec_certs import constants
+from sec_certs.configuration import config
 from sec_certs.utils import parallel_processing
 from sec_certs.utils.tqdm import tqdm
 
diff --git a/src/sec_certs/utils/nvd_dataset_builder.py b/src/sec_certs/utils/nvd_dataset_builder.py
new file mode 100644
index 00000000..5a991f9d
--- /dev/null
+++ b/src/sec_certs/utils/nvd_dataset_builder.py
@@ -0,0 +1,355 @@
+from __future__ import annotations
+
+import itertools
+import logging
+import math
+import random
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from multiprocessing import cpu_count
+from typing import Any, ClassVar, Generic, TypeVar
+
+import numpy as np
+import requests
+from requests import RequestException, Response
+
+from sec_certs import constants
+from sec_certs.dataset.cpe import CPEDataset
+from sec_certs.dataset.cve import CVEDataset
+from sec_certs.utils.parallel_processing import process_parallel
+
+logger = logging.getLogger(__name__)
+
+DatasetType = TypeVar("DatasetType", CPEDataset, CVEDataset, dict)
+
+
+@dataclass
+class NvdDatasetBuilder(Generic[DatasetType], ABC):
+    """
+    Abstract class to build new, or enrich existing, datasets with data from NVD, using their API.
+    Example:
+    ```
+    with CpeNvdDatasetBuilder(api_key=config.nvd_api_key) as builder:
+        cpe_dataset = builder.build_dataset()
+    ```
+    """
+
+    api_key: str | None = None
+    n_threads: int = -1
+    max_attempts: int = 5
+
+    _start_mod_date: datetime | None = field(init=False)
+    _end_mod_date: datetime | None = field(init=False)
+    _ok_responses: list[requests.Response] = field(init=False, default_factory=list)
+    _requests_to_process: list[tuple] = field(init=False, default_factory=list)
+    _attempts_left: int = field(init=False)
+
+    def __post_init__(self):
+        self.clear_state()
+        if not self.api_key:
+            logger.warning("No API key for NVD database was set, the ratelimit is just 5 requests per 30 seconds.")
+
+    def __enter__(self) -> NvdDatasetBuilder:
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.clear_state()
+
+    @property
+    @abstractmethod
+    def RESULTS_PER_PAGE(self):
+        """
+        Specifies "resultsPerPage" parameter to the API
+        """
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def ENDPOINT(self):
+        """
+        Specifies the endpoint, used mostly for logging
+        """
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def ENDPOINT_URL(self):
+        """
+        Specifies the URL to send the requests to
+        """
+        raise NotImplementedError
+
+    def _get_last_update_from_previous_data(self, dataset: DatasetType) -> datetime:
+        """
+        Will retrieve timestamp of the last update from the dataset.
+        """
+        raise NotImplementedError
+
+    @staticmethod
+    def _init_new_dataset() -> DatasetType:
+        """
+        Will initialize new empty dataset.
+        """
+        raise NotImplementedError
+
+    def _process_responses(self, responses: list[Response], dataset_to_fill: DatasetType) -> DatasetType:
+        """
+        Will process the responses, construct objects and fill the `dataset_to_fill`
+        """
+        raise NotImplementedError
+
+    @property
+    def _actual_n_threads(self) -> int:
+        if self.n_threads == -1:
+            return cpu_count()
+        return self.n_threads
+
+    @property
+    def base_params(self) -> dict[str, Any]:
+        dct = {"resultsPerPage": self.RESULTS_PER_PAGE}
+
+        if self._start_mod_date and self._end_mod_date:
+            dct["startModDate"] = self._start_mod_date.isoformat()
+            dct["endModDate"] = self._end_mod_date.isoformat()
+
+        return dct
+
+    @property
+    def headers(self) -> dict[str, Any] | None:
+        dct = {"content-type": "application/json", "User-Agent": "sec-certs"}
+        if self.api_key:
+            dct["apiKey"] = self.api_key
+        return dct
+
+    @property
+    def _base_delay(self) -> int:
+        return 2 if self.api_key else 20
+
+    @staticmethod
+    def fetch_nvd_api(
+        url: str, params: dict[str, Any], headers: dict[str, Any] | None, delay: float = 0
+    ) -> requests.Response:
+        time.sleep(delay)
+        try:
+            response = requests.get(
+                url,
+                params=params,
+                headers=headers,
+                timeout=constants.REQUEST_TIMEOUT,
+            )
+        except requests.exceptions.Timeout:
+            response = requests.Response()
+            response.status_code = 403
+        except Exception:
+            response = requests.Response()
+            response.status_code = 500
+        return response
+
+    def clear_state(self) -> None:
+        """
+        Clears the internal state of the NvdDatasetBuilder. Try to avoid calling this method. Instead, use the class in
+        with statement: `with NvdDatasetBuilder(args) as fetcher: ...`
+        """
+        self._start_mod_date = None
+        self._end_mod_date = None
+        self._ok_responses = []
+        self._requests_to_process = []
+        self._attempts_left = self.max_attempts
+
+    def _fill_in_mod_dates(self, force_full_update: bool, last_update: datetime) -> None:
+        """
+        Decides how to set date arguments in the requests. Effectively this resolves whether the update will be full
+        or incremental.
+
+        :param bool force_full_update: If set to True, will always fetch all data
+        :param datetime last_update: Timestamp of the last update.
+        """
+        if force_full_update:
+            self._start_mod_date = None
+            self._end_mod_date = None
+        else:
+            current_timestamp = datetime.now()
+            # TOCTOU ignored
+            if (current_timestamp - last_update).days >= constants.INCREMENTAL_NVD_UPDATE_MAX_INTERVAL_DAYS:
+                self._start_mod_date = None
+                self._end_mod_date = None
+                logger.info(
+                    f"Will fetch complete {self.ENDPOINT} data from NVD API as the last update was either done >120 days ago, or no previous data was provided."
+                )
+            else:
+                self._start_mod_date = last_update
+                self._end_mod_date = current_timestamp
+
+    def _get_n_total_results(self, fresh: bool = True) -> int:
+        if not fresh:
+            time.sleep(6)
+
+        response = NvdDatasetBuilder.fetch_nvd_api(
+            self.ENDPOINT_URL, params={**self.base_params, **{"resultsPerPage": 0}}, headers=self.headers
+        )
+        if response.status_code == 404:
+            # This is likely due to no CPEs to update, incremental update very soon.
+            return 0
+        if response.status_code != constants.RESPONSE_OK:
+            if fresh:
+                logger.warning(
+                    f"Error when attempting to fetch number of pages to get from NVD API {self.ENDPOINT} endpoint, sleeping 6 seconds and repeating."
+                )
+                return self._get_n_total_results(fresh=False)
+            else:
+                logger.error(
+                    f"Could not fetch the number of pages to get from NVD API {self.ENDPOINT} endpoint even after retry attempt, raising exception."
+                )
+                raise RequestException(
+                    f"Could not fetch the number of pages to get from NVD API {self.ENDPOINT} endpoint even after retry attempt"
+                )
+        return response.json()["totalResults"]
+
+    def _build_arguments(self) -> None:
+        """
+        Makes an API call to NVD API to learn how many records in total will be fetch. Based on that, prepares
+        a list of tuples that parametrize the requests to be made.
+        """
+        n_requests = math.ceil(self._get_n_total_results() / self.RESULTS_PER_PAGE)
+        logger.info(
+            f"Building arguments for NVD requests to {self.ENDPOINT} endpoint. Will send {n_requests} requests."
+        )
+        offsets = [i * self.RESULTS_PER_PAGE for i in range(n_requests)]
+        delays = [self._base_delay * random.randint(1, 3) for _ in range(n_requests)]  # Bulgarian constant
+        self._requests_to_process = [
+            (self.ENDPOINT_URL, {**self.base_params, **{"startIndex": offset}}, self.headers, delay)
+            for offset, delay in zip(offsets, delays)
+        ]
+
+    def _evaluate_responses(self, responses: list[Response]) -> None:
+        """
+        Will fetch successfull responses into self._ok_responses and prune self.requests_to_process accordingly
+        """
+        response_is_nok = np.array([x.status_code != constants.RESPONSE_OK for x in responses])
+        nok_indices = np.where(response_is_nok == True)[0]  # noqa E712, doesn't work with `is True`
+        currently_ok = [x for x in responses if x.status_code == constants.RESPONSE_OK]
+
+        logger.info(
+            f"Attempt {self.max_attempts - self._attempts_left}/{self.max_attempts}: Successfully processed {len(currently_ok)}/{len(self._requests_to_process)} requests."
+        )
+
+        self._ok_responses.extend(currently_ok)
+        self._requests_to_process = [self._requests_to_process[x] for x in nok_indices]
+
+        if self._attempts_left == 0 and self._requests_to_process:
+            logger.warning(
+                f"Failed to process {len(self._requests_to_process)} requests in total, the dataset will be incomplete."
+            )
+
+    def _request_parallel_and_handle_responses(self):
+        """
+        Attempts to fetch the requests in the queue multiple times, and in parallel
+        """
+        if self._attempts_left > 0 and self._requests_to_process:
+            self._attempts_left -= 1
+            self._evaluate_responses(
+                process_parallel(
+                    NvdDatasetBuilder.fetch_nvd_api,
+                    self._requests_to_process,
+                    max_workers=self._actual_n_threads,
+                    unpack=True,
+                    progress_bar_desc=f"Fetching data from {self.ENDPOINT} NVD endpoint",
+                )
+            )
+            self._request_parallel_and_handle_responses()
+
+    def build_dataset(self, dataset_to_fill: DatasetType | None = None, force_full_update: bool = False) -> DatasetType:
+        """
+        Will fetch the resource in a parallelized fashion. If possible, use this within a with statement.
+        E.g., `with NvdDatasetBuilder(args) as builder: builder.build_dataset()`
+        When used outside of the context manager, the caller is responsible for cleaning the state with
+        `self.clear_state()` after running this method.
+
+        :param DatasetType | None dataset_to_fill: Existing dataset to fill-in with new data, defaults to None
+        :param bool force_full_update: If True, will always fetch all data, defaults to False
+        :return DatasetType: Dataset enriched with the new records from NVD.
+        """
+        if dataset_to_fill is None:
+            dataset_to_fill = self._init_new_dataset()
+
+        last_update = self._get_last_update_from_previous_data(dataset_to_fill)
+        self._fill_in_mod_dates(force_full_update, last_update)
+        self._build_arguments()
+        self._request_parallel_and_handle_responses()
+
+        return self._process_responses(self._ok_responses, dataset_to_fill)
+
+
+class CpeNvdDatasetBuilder(NvdDatasetBuilder[CPEDataset]):
+    ENDPOINT: ClassVar[str] = "CPE"
+    ENDPOINT_URL: ClassVar[str] = "https://services.nvd.nist.gov/rest/json/cpes/2.0"
+    RESULTS_PER_PAGE: ClassVar[int] = 10000
+
+    def _process_responses(self, responses: list[requests.Response], cpe_dataset: CPEDataset) -> CPEDataset:
+        products = list(itertools.chain.from_iterable(response.json()["products"] for response in responses))
+        timestamp = self._end_mod_date.isoformat() if self._end_mod_date else responses[-1].json()["timestamp"]
+        cpe_dataset.enhance_with_nvd_data({"timestamp": timestamp, "products": products})
+        return cpe_dataset
+
+    def _get_last_update_from_previous_data(self, previous_data: CPEDataset) -> datetime:
+        return previous_data.last_update_timestamp
+
+    @staticmethod
+    def _init_new_dataset() -> CPEDataset:
+        return CPEDataset()
+
+
+class CveNvdDatasetBuilder(NvdDatasetBuilder[CVEDataset]):
+    ENDPOINT: ClassVar[str] = "CVE"
+    ENDPOINT_URL: ClassVar[str] = "https://services.nvd.nist.gov/rest/json/cves/2.0"
+    RESULTS_PER_PAGE: ClassVar[int] = 2000
+
+    def _process_responses(self, responses: list[Response], cve_dataset: CVEDataset) -> CVEDataset:
+        timestamp = self._end_mod_date.isoformat() if self._end_mod_date else responses[-1].json()["timestamp"]
+        vulns = list(itertools.chain.from_iterable(response.json()["vulnerabilities"] for response in responses))
+        cve_dataset.enhance_with_nvd_data({"timestamp": timestamp, "vulnerabilities": vulns})
+        return cve_dataset
+
+    def _get_last_update_from_previous_data(self, previous_data: CVEDataset) -> datetime:
+        return previous_data.last_update_timestamp
+
+    @staticmethod
+    def _init_new_dataset() -> CVEDataset:
+        return CVEDataset()
+
+
+class CpeMatchNvdDatasetBuilder(NvdDatasetBuilder[dict]):
+    ENDPOINT: ClassVar[str] = "CPEMatch"
+    ENDPOINT_URL: ClassVar[str] = "https://services.nvd.nist.gov/rest/json/cpematch/2.0"
+    RESULTS_PER_PAGE: ClassVar[int] = 5000
+
+    # TODO: I'm actually forgetting to process start_version and end_version
+    def _process_responses(self, responses: list[Response], dataset_to_fill: dict) -> dict:
+        timestamp = self._end_mod_date.isoformat() if self._end_mod_date else responses[-1].json()["timestamp"]
+        match_strings = list(itertools.chain.from_iterable(response.json()["matchStrings"] for response in responses))
+        dataset_to_fill["timestamp"] = timestamp
+
+        inactive_criteria = set()
+        for m in match_strings:
+            if m["matchString"]["status"] == "Inactive":
+                inactive_criteria.add(m["matchString"]["matchCriteriaId"])
+            else:
+                if "matches" in m["matchString"]:
+                    dataset_to_fill["match_strings"][m["matchString"]["matchCriteriaId"]] = {
+                        "criteria": m["matchString"]["criteria"],
+                        "matches": m["matchString"]["matches"],
+                    }
+
+        for inactive in inactive_criteria:
+            dataset_to_fill["match_strings"].pop(inactive, None)
+
+        return dataset_to_fill
+
+    def _get_last_update_from_previous_data(self, previous_data: dict) -> datetime:
+        return datetime.fromisoformat(previous_data["timestamp"])
+
+    @staticmethod
+    def _init_new_dataset() -> dict:
+        return {"timestamp": datetime.fromtimestamp(0).isoformat(), "match_strings": {}}