Added web snapshot and jupyter notebook

3 files changed, 48 insertions, 1 deletions

diff --git a/notebooks/fips_data.ipynb b/notebooks/fips_data.ipynb
new file mode 100644
index 00000000..9a03ff50
--- /dev/null
+++ b/notebooks/fips_data.ipynb
@@ -0,0 +1,31 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "source": [
+    "from sec_certs.dataset.fips import FIPSDataset\n",
+    "\n",
+    "dset: FIPSDataset\n",
+    "dset = FIPSDataset.from_web_latest()"
+   ],
+   "outputs": [],
+   "metadata": {}
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "Dataset is now ready for exploration. Feel free to explore the data."
+   ],
+   "metadata": {}
+  }
+ ],
+ "metadata": {
+  "orig_nbformat": 4,
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
\ No newline at end of file
diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py
index 39cb1a92..7809819a 100644
--- a/sec_certs/dataset/fips.py
+++ b/sec_certs/dataset/fips.py
@@ -1,4 +1,4 @@
-import json
+import tempfile
 import logging
 import os
 from itertools import groupby
@@ -205,6 +205,19 @@ class FIPSDataset(Dataset, ComplexSerializableType):
                 redo=redo,
             )
 
+    @classmethod
+    def from_web_latest(cls):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            dset_path = Path(tmp_dir) / 'fips_latest_dataset.json'
+            logger.info('Downloading the latest FIPS dataset.')
+            helpers.download_file(config.fips_latest_snapshot, dset_path)
+            dset: FIPSDataset = cls.from_json(dset_path)
+            logger.info('The dataset with %s certs and %s algorithms.', len(dset), len(dset.algorithms))
+            logger.info('The dataset does not contain the results of the dependency analysis - calculating them now...')
+            dset.finalize_results()
+            return dset
+
+
     def _append_new_certs_data(self) -> int:
         # we need to know the exact certificates downloaded, so we don't overwrite something already done
         new_certs = self.download_all_htmls()
diff --git a/sec_certs/settings.yaml b/sec_certs/settings.yaml
index f59a882f..a5c8911f 100644
--- a/sec_certs/settings.yaml
+++ b/sec_certs/settings.yaml
@@ -34,4 +34,7 @@ ignore_first_page:
 cert_threshold:
   description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty.
   value: 5
+fips_latest_snapshot:
+  description: Url for the latest snapshot of FIPS dataset
+  value: https://seccerts.org/static/fips_dset_ad8734a39b856ca1a1b7b073713872359bae7545.json