Add a way to get the full MIP and IUT datasets from web.

6 files changed, 52 insertions, 0 deletions

diff --git a/sec_certs/config/settings.yaml b/sec_certs/config/settings.yaml
index 3c9ef2b8..c823181c 100644
--- a/sec_certs/config/settings.yaml
+++ b/sec_certs/config/settings.yaml
@@ -39,9 +39,15 @@ cert_threshold:
 fips_latest_snapshot:
   description: URL for the latest snapshot of FIPS dataset
   value: https://seccerts.org/fips/dataset.json
+fips_iut_dataset:
+  description: URL for the dataset of FIPS IUT data
+  value: https://seccerts.org/fips/iut/dataset.json
 fips_iut_latest_snapshot:
   description: URL for the latest snapshot of FIPS IUT data
   value: https://seccerts.org/fips/iut/latest.json
+fips_mip_dataset:
+  description: URL for the dataset of FIPS MIP data
+  value: https://seccerts.org/fips/mip/dataset.json
 fips_mip_latest_snapshot:
   description: URL for the latest snapshot of FIPS MIP data
   value: https://seccerts.org/fips/mip/latest.json
diff --git a/sec_certs/dataset/fips_iut.py b/sec_certs/dataset/fips_iut.py
index b731115c..791d108c 100644
--- a/sec_certs/dataset/fips_iut.py
+++ b/sec_certs/dataset/fips_iut.py
@@ -1,7 +1,11 @@
 from dataclasses import dataclass
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Dict, Iterator, List, Mapping, Union
 
+import requests
+
+from sec_certs.config.configuration import config
 from sec_certs.dataset.dataset import logger
 from sec_certs.sample.fips_iut import IUTSnapshot
 from sec_certs.serialization.json import ComplexSerializableType
@@ -39,3 +43,15 @@ class IUTDataset(ComplexSerializableType):
     @classmethod
     def from_dict(cls, dct: Mapping) -> "IUTDataset":
         return cls(dct["snapshots"])
+
+    @classmethod
+    def from_web_latest(cls) -> "IUTDataset":
+        """
+        Get the IUTDataset from seccerts.org
+        """
+        iut_resp = requests.get(config.fips_mip_dataset)
+        if iut_resp.status_code != 200:
+            raise ValueError(f"Getting IUT dataset failed: {iut_resp.status_code}")
+        with NamedTemporaryFile() as tmpfile:
+            tmpfile.write(iut_resp.content)
+            return cls.from_json(tmpfile.name)
diff --git a/sec_certs/dataset/fips_mip.py b/sec_certs/dataset/fips_mip.py
index 91d8120c..78455171 100644
--- a/sec_certs/dataset/fips_mip.py
+++ b/sec_certs/dataset/fips_mip.py
@@ -1,7 +1,11 @@
 from dataclasses import dataclass
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Dict, Iterator, List, Mapping, Union
 
+import requests
+
+from sec_certs.config.configuration import config
 from sec_certs.dataset.dataset import logger
 from sec_certs.sample.fips_mip import MIPSnapshot
 from sec_certs.serialization.json import ComplexSerializableType
@@ -39,3 +43,15 @@ class MIPDataset(ComplexSerializableType):
     @classmethod
     def from_dict(cls, dct: Mapping) -> "MIPDataset":
         return cls(dct["snapshots"])
+
+    @classmethod
+    def from_web_latest(cls) -> "MIPDataset":
+        """
+        Get the MIPDataset from seccerts.org
+        """
+        mip_resp = requests.get(config.fips_mip_dataset)
+        if mip_resp.status_code != 200:
+            raise ValueError(f"Getting MIP dataset failed: {mip_resp.status_code}")
+        with NamedTemporaryFile() as tmpfile:
+            tmpfile.write(mip_resp.content)
+            return cls.from_json(tmpfile.name)
diff --git a/tests/data/settings_test.yaml b/tests/data/settings_test.yaml
index 03ddbb62..82da0540 100644
--- a/tests/data/settings_test.yaml
+++ b/tests/data/settings_test.yaml
@@ -39,9 +39,15 @@ cert_threshold:
 fips_latest_snapshot:
   description: URL for the latest snapshot of FIPS dataset
   value: https://seccerts.org/fips/dataset.json
+fips_iut_dataset:
+  description: URL for the dataset of FIPS IUT data
+  value: https://seccerts.org/fips/iut/dataset.json
 fips_iut_latest_snapshot:
   description: URL for the latest snapshot of FIPS IUT data
   value: https://seccerts.org/fips/iut/latest.json
+fips_mip_dataset:
+  description: URL for the dataset of FIPS MIP data
+  value: https://seccerts.org/fips/mip/dataset.json
 fips_mip_latest_snapshot:
   description: URL for the latest snapshot of FIPS MIP data
   value: https://seccerts.org/fips/mip/latest.json
diff --git a/tests/test_fips_iut.py b/tests/test_fips_iut.py
index 61761f27..2d8311c3 100644
--- a/tests/test_fips_iut.py
+++ b/tests/test_fips_iut.py
@@ -15,6 +15,10 @@ class TestFIPSIUT(TestCase):
         self.assertIsNotNone(dset)
         self.assertEqual(len(dset), 2)
 
+    def test_from_dataset_latest(self):
+        dset = IUTDataset.from_web_latest()
+        self.assertIsNotNone(dset)
+
     def test_from_dump(self):
         snap = IUTSnapshot.from_dump(self.test_data_dump)
         self.assertIsNotNone(snap)
diff --git a/tests/test_fips_mip.py b/tests/test_fips_mip.py
index 170cfc4b..2f891035 100644
--- a/tests/test_fips_mip.py
+++ b/tests/test_fips_mip.py
@@ -15,6 +15,10 @@ class TestFIPSMIP(TestCase):
         self.assertIsNotNone(dset)
         self.assertEqual(len(dset), 3)
 
+    def test_from_dataset_latest(self):
+        dset = MIPDataset.from_web_latest()
+        self.assertIsNotNone(dset)
+
     def test_from_dump(self):
         snap = MIPSnapshot.from_dump(self.test_data_dump)
         self.assertIsNotNone(snap)