Add FIPS certificate JSON schema.

6 files changed, 629 insertions, 129 deletions

diff --git a/src/sec_certs/serialization/schemas/__init__.py b/src/sec_certs/serialization/schemas/__init__.py
index e69de29b..31fd252e 100644
--- a/src/sec_certs/serialization/schemas/__init__.py
+++ b/src/sec_certs/serialization/schemas/__init__.py
@@ -0,0 +1,17 @@
+import json
+from importlib import resources
+
+from jsonschema import Draft7Validator
+from referencing import Registry, Resource
+
+_schemas = ["base.json", "cc_certificate.json", "cc_dataset.json", "fips_certificate.json", "fips_dataset.json"]
+
+
+def validator(for_schema: str) -> Draft7Validator:
+    registry = Registry()
+    for schema in _schemas:
+        with resources.open_text("sec_certs.serialization.schemas", schema) as f:
+            schema_json = json.load(f)
+            resource = Resource.from_contents(schema_json)
+            registry = resource @ registry
+    return Draft7Validator(schema={"$ref": for_schema}, registry=registry)
diff --git a/src/sec_certs/serialization/schemas/base.json b/src/sec_certs/serialization/schemas/base.json
new file mode 100644
index 00000000..77fb21fd
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/base.json
@@ -0,0 +1,146 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/base.json",
+  "title": "Base sec-certs definitions",
+  "description": "Common definitions used across sec-certs schemas.",
+  "definitions": {
+    "cve": {
+      "type": "string",
+      "pattern": "^CVE-\\d{4}-\\d{4,}$",
+      "title": "CVE",
+      "description": "Common Vulnerabilities Enumeration (CVE) identifier"
+    },
+    "cpe": {
+      "type": "string",
+      "title": "CPE",
+      "description": "Common Platform Enumeration (CPE) identifier"
+    },
+    "cpe_matches": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "$ref": "#/definitions/cpe"
+      },
+      "uniqueItems": true,
+      "title": "CPE Matches",
+      "description": "List of CPEs (Common Platform Enumerations) associated with the certificate."
+    },
+    "verified_cpe_matches": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "$ref": "#/definitions/cpe"
+      },
+      "uniqueItems": true,
+      "title": "Verified CPE Matches",
+      "description": "List of verified CPEs (Common Platform Enumerations) associated with the certificate."
+    },
+    "related_cves": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "$ref": "#/definitions/cve"
+      },
+      "uniqueItems": true,
+      "title": "Related CVEs",
+      "description": "List of CVEs (Common Vulnerabilities and Exposures) likely associated with the certificate."
+    },
+    "direct_transitive_cves": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "$ref": "#/definitions/cve"
+      },
+      "uniqueItems": true
+    },
+    "indirect_transitive_cves": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "$ref": "#/definitions/cve"
+      },
+      "uniqueItems": true
+    },
+    "extracted_versions": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "uniqueItems": true
+        }
+      },
+      "title": "Extracted Versions",
+      "description": "Product versions extracted from the certification documents."
+    },
+    "document_metadata": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "pdf_file_size_bytes": {
+          "type": "integer",
+          "description": "Size of the PDF document in bytes."
+        },
+        "pdf_is_encrypted": {
+          "type": "boolean",
+          "description": "Indicates if the PDF document is encrypted."
+        },
+        "pdf_number_of_pages": {
+          "type": "integer",
+          "description": "Number of pages in the PDF document."
+        },
+        "pdf_hyperlinks": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "format": "uri"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Set of hyperlinks found in the PDF document."
+        }
+      },
+      "additionalProperties": true
+    },
+    "document_keywords": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "additionalProperties": true,
+      "title": "Certificate Document Keywords",
+      "description": "Extracted keywords from the certification documents.",
+      "$comment": "Consult the rules.yaml for the list of possible keywords and their structure."
+    }
+  }
+}
diff --git a/src/sec_certs/serialization/schemas/cc_certificate.json b/src/sec_certs/serialization/schemas/cc_certificate.json
index 750b4a00..bb6791a1 100644
--- a/src/sec_certs/serialization/schemas/cc_certificate.json
+++ b/src/sec_certs/serialization/schemas/cc_certificate.json
@@ -12,7 +12,10 @@
       "description": "A hex string representing 8 bytes."
     },
     "cc_id": {
-      "type": ["string", "null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "title": "Certificate ID",
       "description": "The unique identifier of the Common Criteria certificate."
     },
@@ -57,43 +60,6 @@
       "title": "Certificate Document State (cert, report, st)",
       "description": "State of the document processing pipeline for cert, report, or st document."
     },
-    "cc_document_metadata": {
-      "type": [
-        "object",
-        "null"
-      ],
-      "properties": {
-        "pdf_file_size_bytes": {
-          "type": "integer"
-        },
-        "pdf_is_encrypted": {
-          "type": "boolean"
-        },
-        "pdf_number_of_pages": {
-          "type": "integer"
-        },
-        "pdf_hyperlinks": {
-          "type": [
-            "object",
-            "null"
-          ],
-          "properties": {
-            "_type": {
-              "const": "Set"
-            },
-            "elements": {
-              "type": "array",
-              "items": {
-                "type": "string",
-                "format": "uri"
-              },
-              "uniqueItems": true
-            }
-          }
-        }
-      },
-      "additionalProperties": true
-    },
     "cc_document_frontpage_body": {
       "type": "object",
       "properties": {
@@ -219,16 +185,6 @@
       },
       "additionalProperties": false
     },
-    "cc_document_keywords": {
-      "type": [
-        "object",
-        "null"
-      ],
-      "additionalProperties": true,
-      "title": "Certificate Document Keywords",
-      "description": "Extracted keywords from the certification documents.",
-      "$comment": "Consult the rules.yaml for the list of possible keywords and their structure."
-    },
     "cc_references": {
       "type": "object",
       "properties": {
@@ -336,17 +292,6 @@
       "additionalProperties": false,
       "title": "SAR",
       "description": "Security Assurance Requirement (SAR) as defined in Common Criteria."
-    },
-    "cve": {
-      "type": "string",
-      "pattern": "^CVE-\\d{4}-\\d{4,}$",
-      "title": "CVE",
-      "description": "Common Vulnerabilities Enumeration (CVE) identifier"
-    },
-    "cpe": {
-      "type": "string",
-      "title": "CPE",
-      "description": "Common Platform Enumeration (CPE) identifier"
     }
   },
   "properties": {
@@ -512,60 +457,19 @@
           "description": "The lab/scheme that issued the certificate."
         },
         "cpe_matches": {
-          "type": [
-            "array",
-            "null"
-          ],
-          "items": {
-            "$ref": "#/definitions/cve"
-          },
-          "uniqueItems": true,
-          "title": "CPE Matches",
-          "description": "List of CPEs (Common Platform Enumerations) associated with the certificate."
+          "$ref": "/schemas/base.json#/definitions/cpe_matches"
         },
         "verified_cpe_matches": {
-          "type": [
-            "array",
-            "null"
-          ],
-          "items": {
-            "$ref": "#/definitions/cve"
-          },
-          "uniqueItems": true,
-          "title": "Verified CPE Matches",
-          "description": "List of verified CPEs (Common Platform Enumerations) associated with the certificate."
+          "$ref": "/schemas/base.json#/definitions/verified_cpe_matches"
         },
         "related_cves": {
-          "type": [
-            "array",
-            "null"
-          ],
-          "items": {
-            "$ref": "#/definitions/cve"
-          },
-          "uniqueItems": true,
-          "title": "Related CVEs",
-          "description": "List of CVEs (Common Vulnerabilities and Exposures) likely associated with the certificate."
+          "$ref": "/schemas/base.json#/definitions/related_cves"
         },
         "direct_transitive_cves": {
-          "type": [
-            "array",
-            "null"
-          ],
-          "items": {
-            "$ref": "#/definitions/cve"
-          },
-          "uniqueItems": true
+          "$ref": "/schemas/base.json#/definitions/direct_transitive_cves"
         },
         "indirect_transitive_cves": {
-          "type": [
-            "array",
-            "null"
-          ],
-          "items": {
-            "$ref": "#/definitions/cve"
-          },
-          "uniqueItems": true
+          "$ref": "/schemas/base.json#/definitions/indirect_transitive_cves"
         },
         "extracted_sars": {
           "type": [
@@ -588,24 +492,7 @@
           "description": "Security Assurance Requirements (SARs) extracted from the certification documents."
         },
         "extracted_versions": {
-          "type": [
-            "object",
-            "null"
-          ],
-          "properties": {
-            "_type": {
-              "const": "Set"
-            },
-            "elements": {
-              "type": "array",
-              "items": {
-                "type": "string"
-              },
-              "uniqueItems": true
-            }
-          },
-          "title": "Extracted Versions",
-          "description": "Product versions extracted from the certification documents."
+          "$ref": "/schemas/base.json#/definitions/extracted_versions"
         },
         "prev_certificates": {
           "type": [
@@ -689,22 +576,22 @@
           "const": "sec_certs.sample.cc.CCCertificate.PdfData"
         },
         "report_metadata": {
-          "$ref": "#/definitions/cc_document_metadata",
+          "$ref": "/schemas/base.json#/definitions/document_metadata",
           "title": "Report Metadata",
           "description": "Metadata extracted from the certification report PDF document file."
         },
         "st_metadata": {
-          "$ref": "#/definitions/cc_document_metadata",
+          "$ref": "/schemas/base.json#/definitions/document_metadata",
           "title": "ST Metadata",
           "description": "Metadata extracted from the security target PDF document file."
         },
         "report_keywords": {
-          "$ref": "#/definitions/cc_document_keywords",
+          "$ref": "/schemas/base.json#/definitions/document_keywords",
           "title": "Report Keywords",
           "description": "Keywords extracted from the certification report document."
         },
         "st_keywords": {
-          "$ref": "#/definitions/cc_document_keywords",
+          "$ref": "/schemas/base.json#/definitions/document_keywords",
           "title": "ST Keywords",
           "description": "Keywords extracted from the security target document."
         }
diff --git a/src/sec_certs/serialization/schemas/cc_dataset.json b/src/sec_certs/serialization/schemas/cc_dataset.json
index 51a07b72..3e57b349 100644
--- a/src/sec_certs/serialization/schemas/cc_dataset.json
+++ b/src/sec_certs/serialization/schemas/cc_dataset.json
@@ -53,9 +53,10 @@
     "certs": {
       "type": "array",
       "items": {
-        "$ref": "/schemas/cc_certificate.json"
+        "$ref": "/schemas/cc_certificate.json#"
       },
       "title": "List of certificates in the dataset"
     }
-  }
+  },
+  "additionalProperties": false
 }
\ No newline at end of file
diff --git a/src/sec_certs/serialization/schemas/fips_certificate.json b/src/sec_certs/serialization/schemas/fips_certificate.json
index e69de29b..9356fd27 100644
--- a/src/sec_certs/serialization/schemas/fips_certificate.json
+++ b/src/sec_certs/serialization/schemas/fips_certificate.json
@@ -0,0 +1,387 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/fips_certificate.json",
+  "title": "FIPS 140 Certificate",
+  "description": "Schema for a FIPS 140 certificate.",
+  "type": "object",
+  "definitions": {
+    "fips_dgst": {
+      "type": "string",
+      "pattern": "^[0-9a-fA-F]{16}$",
+      "title": "Certificate Digest",
+      "description": "A hex string representing 8 bytes."
+    },
+    "fips_id": {
+      "type": "integer",
+      "minimum": 0,
+      "title": "Certificate ID",
+      "description": "A non-negative integer representing the certificate ID."
+    },
+    "fips_references": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.certificate.References"
+        },
+        "directly_referenced_by": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/definitions/fips_id"
+          },
+          "uniqueItems": true,
+          "description": "Certificates that directly reference this certificate."
+        },
+        "indirectly_referenced_by": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/definitions/fips_id"
+          },
+          "uniqueItems": true,
+          "description": "Certificates that indirectly reference this certificate."
+        },
+        "directly_referencing": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/definitions/fips_id"
+          },
+          "uniqueItems": true,
+          "description": "Certificates that are directly referenced by this certificate."
+        },
+        "indirectly_referencing": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/definitions/fips_id"
+          },
+          "uniqueItems": true,
+          "description": "Certificates that are indirectly referenced by this certificate."
+        }
+      }
+    }
+  },
+  "properties": {
+    "_type": {
+      "const": "sec_certs.sample.fips.FIPSCertificate"
+    },
+    "dgst": {
+      "$ref": "#/definitions/fips_dgst"
+    },
+    "cert_id": {
+      "$ref": "#/definitions/fips_id"
+    },
+    "web_data": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.WebData"
+        },
+        "module_name": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Module Name",
+          "description": "The name of the certified module."
+        },
+        "validation_history": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "type": "object",
+            "properties": {
+              "_type": {
+                "const": "sec_certs.sample.fips.FIPSCertificate.ValidationHistoryEntry"
+              },
+              "date": {
+                "type": "string",
+                "format": "date",
+                "description": "The date of the validation event in YYYY-MM-DD format."
+              },
+              "validation_type": {
+                "enum": [
+                  "Initial",
+                  "Update"
+                ],
+                "description": "The type of validation event."
+              },
+              "lab": {
+                "type": "string",
+                "description": "The name of the lab that performed the validation."
+              }
+            },
+            "additionalProperties": false
+          }
+        },
+        "vendor_url": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "vendor": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "certificate_pdf_url": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "module_type": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "standard": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "status": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "level": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "caveat": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "exceptions": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "embodiment": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "description": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "tested_conf": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "hw_versions": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "fw_versions": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "sw_versions": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "mentioned_certs": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "historical_reason": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "date_sunset": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "revoked_reason": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "revoked_link": {
+          "type": [
+            "string",
+            "null"
+          ]
+        }
+      }
+    },
+    "pdf_data": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.PdfData"
+        },
+        "keywords": {
+          "$ref": "/schemas/base.json#/definitions/document_keywords",
+          "title": "Security Policy Keywords",
+          "description": "Keywords extracted from the security policy document."
+        },
+        "policy_metadata": {
+          "$ref": "/schemas/base.json#/definitions/document_metadata",
+          "title": "Security Policy Metadata",
+          "description": "Metadata extracted from the security policy document."
+        }
+      }
+    },
+    "heuristics": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.Heuristics"
+        },
+        "algorithms": {
+          "type": "object",
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
+        },
+        "extracted_versions": {
+          "$ref": "/schemas/base.json#/definitions/extracted_versions"
+        },
+        "cpe_matches": {
+          "$ref": "/schemas/base.json#/definitions/cpe_matches"
+        },
+        "verified_cpe_matches": {
+          "$ref": "/schemas/base.json#/definitions/verified_cpe_matches"
+        },
+        "related_cves": {
+          "$ref": "/schemas/base.json#/definitions/related_cves"
+        },
+        "policy_prunned_references": {
+          "type": "object",
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_id"
+              },
+              "uniqueItems": true
+            }
+          }
+        },
+        "module_prunned_references": {
+          "type": "object",
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_id"
+              },
+              "uniqueItems": true
+            }
+          }
+        },
+        "policy_processed_references": {
+          "$ref": "#/definitions/fips_references",
+          "description": "References to and from other certificates, based on the security policy document."
+        },
+        "module_processed_references": {
+          "$ref": "#/definitions/fips_references",
+          "description": "References to and from other certificates, based on the module page."
+        },
+        "direct_transitive_cves": {
+          "$ref": "/schemas/base.json#/definitions/direct_transitive_cves"
+        },
+        "indirect_transitive_cves": {
+          "$ref": "/schemas/base.json#/definitions/indirect_transitive_cves"
+        }
+      },
+      "additionalProperties": false,
+      "title": "Certificate Heuristics",
+      "description": "Heuristic data extracted from the certification documents."
+    },
+    "state": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.InternalState"
+        },
+        "module_download_ok": {
+          "type": "boolean"
+        },
+        "policy_download_ok": {
+          "type": "boolean"
+        },
+        "policy_convert_garbage": {
+          "type": "boolean"
+        },
+        "policy_convert_ok": {
+          "type": "boolean"
+        },
+        "module_extract_ok": {
+          "type": "boolean"
+        },
+        "policy_extract_ok": {
+          "type": "boolean"
+        },
+        "policy_pdf_hash": {
+          "type": "string",
+          "pattern": "^[0-9a-fA-F]{64}$",
+          "description": "SHA256 hash of the policy PDF file."
+        },
+        "policy_txt_hash": {
+          "type": "string",
+          "pattern": "^[0-9a-fA-F]{64}$",
+          "description": "SHA256 hash of the policy txt file."
+        }
+      }
+    }
+  },
+  "additionalProperties": false
+}
\ No newline at end of file
diff --git a/src/sec_certs/serialization/schemas/fips_dataset.json b/src/sec_certs/serialization/schemas/fips_dataset.json
new file mode 100644
index 00000000..c488bce6
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/fips_dataset.json
@@ -0,0 +1,62 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/fips_dataset.json",
+  "title": "FIPS 140 Dataset",
+  "description": "Schema for a FIPS 140 dataset.",
+  "type": "object",
+  "properties": {
+    "_type": {
+      "const": "sec_certs.dataset.fips.FIPSDataset"
+    },
+    "state": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.dataset.dataset.Dataset.DatasetInternalState"
+        },
+        "meta_sources_parsed": {
+          "type": "boolean"
+        },
+        "artifacts_downloaded": {
+          "type": "boolean"
+        },
+        "pdfs_converted": {
+          "type": "boolean"
+        },
+        "auxiliary_datasets_processed": {
+          "type": "boolean"
+        },
+        "certs_analyzed": {
+          "type": "boolean"
+        }
+      },
+      "title": "Dataset internal state"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "title": "Timestamp of the dataset creation"
+    },
+    "name": {
+      "type": "string",
+      "title": "Name of the dataset"
+    },
+    "description": {
+      "type": "string",
+      "title": "Description of the dataset"
+    },
+    "n_certs": {
+      "type": "integer",
+      "minimum": 0,
+      "title": "Number of certificates in the dataset"
+    },
+    "certs": {
+      "type": "array",
+      "items": {
+        "$ref": "/schemas/fips_certificate.json#"
+      },
+      "title": "List of certificates in the dataset"
+    }
+  },
+  "additionalProperties": false
+}
\ No newline at end of file