Merge pull request #516 from crocs-muni/feat/jsonschema

Make a JSON schema for CC Certificate and Dataset objects.

6 files changed, 1479 insertions, 0 deletions

diff --git a/src/sec_certs/serialization/schemas/__init__.py b/src/sec_certs/serialization/schemas/__init__.py
new file mode 100644
index 00000000..31fd252e
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/__init__.py
@@ -0,0 +1,17 @@
+import json
+from importlib import resources
+
+from jsonschema import Draft7Validator
+from referencing import Registry, Resource
+
+_schemas = ["base.json", "cc_certificate.json", "cc_dataset.json", "fips_certificate.json", "fips_dataset.json"]
+
+
+def validator(for_schema: str) -> Draft7Validator:
+    registry = Registry()
+    for schema in _schemas:
+        with resources.open_text("sec_certs.serialization.schemas", schema) as f:
+            schema_json = json.load(f)
+            resource = Resource.from_contents(schema_json)
+            registry = resource @ registry
+    return Draft7Validator(schema={"$ref": for_schema}, registry=registry)
diff --git a/src/sec_certs/serialization/schemas/base.json b/src/sec_certs/serialization/schemas/base.json
new file mode 100644
index 00000000..6d5861d8
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/base.json
@@ -0,0 +1,192 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/base.json",
+  "title": "Base sec-certs definitions",
+  "description": "Common definitions used across sec-certs schemas.",
+  "definitions": {
+    "cve": {
+      "type": "string",
+      "pattern": "^CVE-\\d{4}-\\d{4,}$",
+      "title": "CVE",
+      "description": "Common Vulnerabilities Enumeration (CVE) identifier"
+    },
+    "cpe": {
+      "type": "string",
+      "title": "CPE",
+      "description": "Common Platform Enumeration (CPE) identifier"
+    },
+    "cpe_matches": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/cpe"
+          },
+          "uniqueItems": true
+        }
+      },
+      "title": "CPE Matches",
+      "description": "List of CPEs (Common Platform Enumerations) associated with the certificate."
+    },
+    "verified_cpe_matches": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/cpe"
+          },
+          "uniqueItems": true
+        }
+      },
+      "title": "Verified CPE Matches",
+      "description": "List of verified CPEs (Common Platform Enumerations) associated with the certificate."
+    },
+    "related_cves": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/cve"
+          },
+          "uniqueItems": true
+        }
+      },
+      "title": "Related CVEs",
+      "description": "List of CVEs (Common Vulnerabilities and Exposures) likely associated with the certificate."
+    },
+    "direct_transitive_cves": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/cve"
+          },
+          "uniqueItems": true
+        }
+      }
+    },
+    "indirect_transitive_cves": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/cve"
+          },
+          "uniqueItems": true
+        }
+      }
+    },
+    "dgst": {
+      "type": "string",
+      "pattern": "^[0-9a-fA-F]{16}$",
+      "title": "Certificate Digest",
+      "description": "A hex string representing 8 bytes."
+    },
+    "extracted_versions": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "uniqueItems": true
+        }
+      },
+      "title": "Extracted Versions",
+      "description": "Product versions extracted from the certification documents."
+    },
+    "document_metadata": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "pdf_file_size_bytes": {
+          "type": "integer",
+          "description": "Size of the PDF document in bytes."
+        },
+        "pdf_is_encrypted": {
+          "type": "boolean",
+          "description": "Indicates if the PDF document is encrypted."
+        },
+        "pdf_number_of_pages": {
+          "type": "integer",
+          "description": "Number of pages in the PDF document."
+        },
+        "pdf_hyperlinks": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "format": "uri"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Set of hyperlinks found in the PDF document."
+        }
+      },
+      "additionalProperties": true
+    },
+    "document_keywords": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "additionalProperties": true,
+      "title": "Certificate Document Keywords",
+      "description": "Extracted keywords from the certification documents.",
+      "$comment": "Consult the rules.yaml for the list of possible keywords and their structure."
+    }
+  }
+}
diff --git a/src/sec_certs/serialization/schemas/cc_certificate.json b/src/sec_certs/serialization/schemas/cc_certificate.json
new file mode 100644
index 00000000..1bba5b6f
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/cc_certificate.json
@@ -0,0 +1,634 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/cc_certificate.json",
+  "title": "CC Certificate",
+  "description": "Schema for a Common Criteria (CC) certificate.",
+  "type": "object",
+  "definitions": {
+    "cc_id": {
+      "type": [
+        "string",
+        "null"
+      ],
+      "title": "Certificate ID",
+      "description": "The unique identifier of the Common Criteria certificate."
+    },
+    "cc_document_state": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.document_state.DocumentState"
+        },
+        "convert_garbage": {
+          "type": "boolean",
+          "description": "Whether the document was converted to garbage (e.g., empty or invalid content)."
+        },
+        "convert_ok": {
+          "type": "boolean",
+          "description": "Whether the document was successfully converted from PDF to text."
+        },
+        "download_ok": {
+          "type": "boolean",
+          "description": "Whether the document was successfully downloaded."
+        },
+        "extract_ok": {
+          "type": "boolean",
+          "description": "Whether keyword extraction from the document was successful."
+        },
+        "pdf_hash": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "description": "SHA256 hash of the PDF document, if available."
+        },
+        "txt_hash": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "description": "SHA256 hash of the text document, if available."
+        }
+      },
+      "additionalProperties": false,
+      "title": "Certificate Document State (cert, report, st)",
+      "description": "State of the document processing pipeline for cert, report, or st document."
+    },
+    "cc_document_frontpage_body": {
+      "type": "object",
+      "properties": {
+        "match_rules": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "List of matching rules that were triggered to identify the front page."
+        },
+        "cert_id": {
+          "$ref": "#/definitions/cc_id",
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Certificate ID",
+          "description": "The unique identifier of the Common Criteria certificate."
+        },
+        "cert_item": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Certified Item",
+          "description": "The certified item as extracted from the front page."
+        },
+        "developer": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Developer",
+          "description": "The developer of the certified item as extracted from the front page."
+        },
+        "cert_lab": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Certification Lab",
+          "description": "The lab/scheme that issued the certificate as extracted from the front page."
+        },
+        "ref_protection_profiles": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "cc_version": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "CC Version",
+          "description": "The version of the Common Criteria standard as extracted from the front page."
+        },
+        "cc_security_level": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "CC Security Level",
+          "description": "The security level (e.g., EAL) as extracted from the front page."
+        }
+      },
+      "additionalProperties": false
+    },
+    "cc_document_frontpage": {
+      "type": "object",
+      "properties": {
+        "AU": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "CA": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "FR": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "DE": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "IN": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "IT": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "JP": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "MY": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "NL": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "NO": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "KR": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "PL": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "SG": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "ES": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "SE": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "TR": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        },
+        "US": {
+          "$ref": "#/definitions/cc_document_frontpage_body"
+        }
+      },
+      "additionalProperties": false
+    },
+    "cc_references": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.certificate.References"
+        },
+        "directly_referenced_by": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/cc_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that directly reference this certificate."
+        },
+        "directly_referencing": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/cc_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that are directly referenced by this certificate."
+        },
+        "indirectly_referenced_by": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/cc_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that indirectly reference this certificate."
+        },
+        "indirectly_referencing": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/cc_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that are indirectly referenced by this certificate."
+        }
+      },
+      "additionalProperties": false,
+      "title": "Certificate References (report, st)",
+      "description": "References to and from other certificates, based on a document."
+    },
+    "cc_sar": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.sar.SAR"
+        },
+        "family": {
+          "type": "string",
+          "title": "SAR Family",
+          "description": "The family of the Security Assurance Requirement (SAR)."
+        },
+        "level": {
+          "type": "integer",
+          "title": "SAR Level",
+          "description": "The level of the Security Assurance Requirement (SAR)."
+        }
+      },
+      "additionalProperties": false,
+      "title": "SAR",
+      "description": "Security Assurance Requirement (SAR) as defined in Common Criteria."
+    }
+  },
+  "properties": {
+    "_type": {
+      "const": "sec_certs.sample.cc.CCCertificate"
+    },
+    "name": {
+      "type": "string",
+      "title": "Certificate Name",
+      "description": "The name of the Common Criteria certificate."
+    },
+    "category": {
+      "enum": [
+        "Access Control Devices and Systems",
+        "Boundary Protection Devices and Systems",
+        "Data Protection",
+        "Databases",
+        "Detection Devices and Systems",
+        "ICs, Smart Cards and Smart Card-Related Devices and Systems",
+        "Key Management Systems",
+        "Mobility",
+        "Multi-Function Devices",
+        "Network and Network-Related Devices and Systems",
+        "Operating Systems",
+        "Other Devices and Systems",
+        "Products for Digital Signatures",
+        "Trusted Computing",
+        "Biometric Systems and Devices"
+      ],
+      "title": "Certificate Category",
+      "description": "The category of the Common Criteria certificate."
+    },
+    "scheme": {
+      "enum": [
+        "AU",
+        "CA",
+        "FR",
+        "DE",
+        "IN",
+        "IT",
+        "JP",
+        "MY",
+        "NL",
+        "NO",
+        "KR",
+        "PL",
+        "SG",
+        "ES",
+        "SE",
+        "TR",
+        "US",
+        "UK",
+        "QA"
+      ],
+      "title": "Certification Scheme",
+      "description": "The certification scheme under which the certificate was issued."
+    },
+    "status": {
+      "enum": [
+        "active",
+        "archived"
+      ],
+      "title": "Certificate Status",
+      "description": "The current status of the Common Criteria certificate."
+    },
+    "not_valid_after": {
+      "type": "string",
+      "format": "date",
+      "title": "Expiry Date",
+      "description": "The date until which the certificate is valid (expiry date)."
+    },
+    "not_valid_before": {
+      "type": "string",
+      "format": "date",
+      "title": "Certification Date",
+      "description": "The date from which the certificate is valid (certification date)."
+    },
+    "cert_link": {
+      "type": [
+        "string",
+        "null"
+      ],
+      "format": "uri",
+      "title": "Certificate Link",
+      "description": "A URL linking to the official certificate document."
+    },
+    "report_link": {
+      "type": [
+        "string",
+        "null"
+      ],
+      "format": "uri",
+      "title": "Certification Report Link",
+      "description": "A URL linking to the certification report document."
+    },
+    "st_link": {
+      "type": [
+        "string",
+        "null"
+      ],
+      "format": "uri",
+      "title": "Security Target Link",
+      "description": "A URL linking to the security target document."
+    },
+    "manufacturer": {
+      "type": [
+        "string",
+        "null"
+      ],
+      "title": "Manufacturer",
+      "description": "The name of the manufacturer of the certified product."
+    },
+    "manufacturer_web": {
+      "type": [
+        "string",
+        "null"
+      ],
+      "format": "uri",
+      "title": "Manufacturer Website",
+      "description": "The website URL of the manufacturer."
+    },
+    "security_level": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "uniqueItems": true
+        }
+      },
+      "title": "Security Level",
+      "description": "The security levels associated with the certificate."
+    },
+    "dgst": {
+      "$ref": "base.json#/definitions/dgst"
+    },
+    "heuristics": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.cc.CCCertificate.Heuristics"
+        },
+        "eal": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "annotated_references": {
+          "type": "null"
+        },
+        "cert_id": {
+          "$ref": "#/definitions/cc_id",
+          "title": "Certificate ID",
+          "description": "The unique identifier of the Common Criteria certificate."
+        },
+        "cert_lab": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "type": "string"
+          },
+          "title": "Certification Lab",
+          "description": "The lab/scheme that issued the certificate."
+        },
+        "cpe_matches": {
+          "$ref": "base.json#/definitions/cpe_matches"
+        },
+        "verified_cpe_matches": {
+          "$ref": "base.json#/definitions/verified_cpe_matches"
+        },
+        "related_cves": {
+          "$ref": "base.json#/definitions/related_cves"
+        },
+        "direct_transitive_cves": {
+          "$ref": "base.json#/definitions/direct_transitive_cves"
+        },
+        "indirect_transitive_cves": {
+          "$ref": "base.json#/definitions/indirect_transitive_cves"
+        },
+        "extracted_sars": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/cc_sar"
+              },
+              "uniqueItems": true
+            }
+          },
+          "title": "Extracted SARs",
+          "description": "Security Assurance Requirements (SARs) extracted from the certification documents."
+        },
+        "extracted_versions": {
+          "$ref": "base.json#/definitions/extracted_versions"
+        },
+        "prev_certificates": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/definitions/cc_id"
+          },
+          "title": "Previous Certificates",
+          "description": "List of previous versions of the certificate in a series, if applicable."
+        },
+        "next_certificates": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "$ref": "#/definitions/cc_id"
+          },
+          "title": "Next Certificates",
+          "description": "List of next versions of the certificate in a series, if applicable."
+        },
+        "report_references": {
+          "$ref": "#/definitions/cc_references",
+          "title": "Report References",
+          "description": "References to and from other certificates, based on the certification report document."
+        },
+        "scheme_data": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "title": "Scheme Data",
+          "description": "Scheme-specific data extracted from the scheme websites.",
+          "$comment": "This is a free-form object and can contain any properties."
+        },
+        "st_references": {
+          "$ref": "#/definitions/cc_references",
+          "title": "ST References",
+          "description": "References to and from other certificates, based on the security target document."
+        },
+        "protection_profiles": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "base.json#/definitions/dgst"
+              },
+              "uniqueItems": true
+            }
+          }
+        }
+      },
+      "additionalProperties": false,
+      "title": "Certificate Heuristics",
+      "description": "Heuristic data extracted from the certification documents."
+    },
+    "maintenance_updates": {
+      "type": "object"
+    },
+    "protection_profile_links": {
+      "type": ["object", "null"],
+      "properties": {
+        "_type": {
+          "const": "Set"
+        },
+        "elements": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "format": "uri"
+          },
+          "uniqueItems": true
+        }
+      }
+    },
+    "pdf_data": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.cc.CCCertificate.PdfData"
+        },
+        "report_metadata": {
+          "$ref": "base.json#/definitions/document_metadata",
+          "title": "Report Metadata",
+          "description": "Metadata extracted from the certification report PDF document file."
+        },
+        "st_metadata": {
+          "$ref": "base.json#/definitions/document_metadata",
+          "title": "ST Metadata",
+          "description": "Metadata extracted from the security target PDF document file."
+        },
+        "report_keywords": {
+          "$ref": "base.json#/definitions/document_keywords",
+          "title": "Report Keywords",
+          "description": "Keywords extracted from the certification report document."
+        },
+        "st_keywords": {
+          "$ref": "base.json#/definitions/document_keywords",
+          "title": "ST Keywords",
+          "description": "Keywords extracted from the security target document."
+        }
+      },
+      "title": "Extracted PDF Data",
+      "description": "Data extracted from the certification PDF documents."
+    },
+    "state": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.cc.CCCertificate.InternalState"
+        },
+        "cert": {
+          "$ref": "#/definitions/cc_document_state"
+        },
+        "report": {
+          "$ref": "#/definitions/cc_document_state"
+        },
+        "st": {
+          "$ref": "#/definitions/cc_document_state"
+        }
+      },
+      "additionalProperties": false,
+      "title": "Certificate Document States",
+      "description": "State of the document processing pipeline for cert, report, and st documents."
+    }
+  },
+  "additionalProperties": false
+}
diff --git a/src/sec_certs/serialization/schemas/cc_dataset.json b/src/sec_certs/serialization/schemas/cc_dataset.json
new file mode 100644
index 00000000..1544c619
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/cc_dataset.json
@@ -0,0 +1,62 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/cc_dataset.json",
+  "title": "CC Dataset",
+  "description": "Schema for a Common Criteria (CC) dataset.",
+  "type": "object",
+  "properties": {
+    "_type": {
+      "const": "sec_certs.dataset.cc.CCDataset"
+    },
+    "state": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.dataset.dataset.Dataset.DatasetInternalState"
+        },
+        "meta_sources_parsed": {
+          "type": "boolean"
+        },
+        "artifacts_downloaded": {
+          "type": "boolean"
+        },
+        "pdfs_converted": {
+          "type": "boolean"
+        },
+        "auxiliary_datasets_processed": {
+          "type": "boolean"
+        },
+        "certs_analyzed": {
+          "type": "boolean"
+        }
+      },
+      "title": "Dataset internal state"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "title": "Timestamp of the dataset creation"
+    },
+    "name": {
+      "type": "string",
+      "title": "Name of the dataset"
+    },
+    "description": {
+      "type": "string",
+      "title": "Description of the dataset"
+    },
+    "n_certs": {
+      "type": "integer",
+      "minimum": 0,
+      "title": "Number of certificates in the dataset"
+    },
+    "certs": {
+      "type": "array",
+      "items": {
+        "$ref": "cc_certificate.json#"
+      },
+      "title": "List of certificates in the dataset"
+    }
+  },
+  "additionalProperties": false
+}
diff --git a/src/sec_certs/serialization/schemas/fips_certificate.json b/src/sec_certs/serialization/schemas/fips_certificate.json
new file mode 100644
index 00000000..e9961262
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/fips_certificate.json
@@ -0,0 +1,512 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/fips_certificate.json",
+  "title": "FIPS 140 Certificate",
+  "description": "Schema for a FIPS 140 certificate.",
+  "type": "object",
+  "definitions": {
+    "fips_dgst": {
+      "type": "string",
+      "pattern": "^[0-9a-fA-F]{16}$",
+      "title": "Certificate Digest",
+      "description": "A hex string representing 8 bytes."
+    },
+    "fips_id": {
+      "type": "integer",
+      "minimum": 0,
+      "title": "Certificate ID",
+      "description": "A non-negative integer representing the certificate ID."
+    },
+    "fips_str_id": {
+      "type": "string",
+      "pattern": "^[0-9]+$",
+      "title": "Certificate ID",
+      "description": "A non-negative integer representing the certificate ID, in string form."
+    },
+    "fips_references": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.certificate.References"
+        },
+        "directly_referenced_by": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_str_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that directly reference this certificate."
+        },
+        "indirectly_referenced_by": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_str_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that indirectly reference this certificate."
+        },
+        "directly_referencing": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_str_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that are directly referenced by this certificate."
+        },
+        "indirectly_referencing": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_str_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "description": "Certificates that are indirectly referenced by this certificate."
+        }
+      }
+    }
+  },
+  "properties": {
+    "_type": {
+      "const": "sec_certs.sample.fips.FIPSCertificate"
+    },
+    "dgst": {
+      "$ref": "#/definitions/fips_dgst"
+    },
+    "cert_id": {
+      "$ref": "#/definitions/fips_id"
+    },
+    "web_data": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.WebData"
+        },
+        "module_name": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Module Name",
+          "description": "The name of the certified module."
+        },
+        "validation_history": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "type": "object",
+            "properties": {
+              "_type": {
+                "const": "sec_certs.sample.fips.FIPSCertificate.ValidationHistoryEntry"
+              },
+              "date": {
+                "type": "string",
+                "format": "date",
+                "description": "The date of the validation event in YYYY-MM-DD format."
+              },
+              "validation_type": {
+                "enum": [
+                  "Initial",
+                  "Update"
+                ],
+                "description": "The type of validation event."
+              },
+              "lab": {
+                "type": "string",
+                "description": "The name of the lab that performed the validation."
+              }
+            },
+            "additionalProperties": false
+          }
+        },
+        "vendor_url": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "format": "uri",
+          "title": "Vendor URL",
+          "description": "The URL of the vendor's website."
+        },
+        "vendor": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Vendor Name",
+          "description": "The name of the vendor of the module."
+        },
+        "certificate_pdf_url": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "format": "uri",
+          "title": "Certificate PDF URL",
+          "description": "The URL of the certificate PDF document."
+        },
+        "module_type": {
+          "enum": [
+            "Firmware",
+            "Firmware-Hybrid",
+            "Hardware",
+            "Software",
+            "Software-Hybrid",
+            "Hybrid",
+            null
+          ],
+          "title": "Certified Module Type",
+          "description": "The type of the certified module."
+        },
+        "standard": {
+          "enum": [
+            "FIPS 140-1",
+            "FIPS 140-2",
+            "FIPS 140-3",
+            null
+          ],
+          "title": "FIPS Standard",
+          "description": "The FIPS standard under which the module was certified."
+        },
+        "status": {
+          "enum": [
+            "active",
+            "historical",
+            "revoked",
+            null
+          ],
+          "title": "Certificate Status",
+          "description": "The current status of the certificate."
+        },
+        "level": {
+          "type": [
+            "integer",
+            "null"
+          ],
+          "minimum": 1,
+          "maximum": 4,
+          "title": "Security Level",
+          "description": "The security level of the certified module (1 to 4)."
+        },
+        "caveat": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Caveat",
+          "description": "Any caveats associated with the certification."
+        },
+        "exceptions": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "type": "string"
+          },
+          "title": "Exceptions",
+          "description": "Any exceptions associated with the certification."
+        },
+        "embodiment": {
+          "anyOf": [
+            {
+              "enum": [
+                "Multi-Chip Stand Alone",
+                "Multi-Chip Embedded",
+                "Single Chip",
+                "*"
+              ]
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "title": "Embodiment",
+          "description": "The embodiment of the certified module."
+        },
+        "description": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Module Description",
+          "description": "A brief description of the certified module."
+        },
+        "tested_conf": {
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "type": "string"
+          },
+          "title": "Tested Configurations",
+          "description": "The tested configuration(s) of the certified module."
+        },
+        "hw_versions": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Hardware Versions",
+          "description": "The hardware version(s) of the certified module."
+        },
+        "fw_versions": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Firmware Versions",
+          "description": "The firmware version(s) of the certified module."
+        },
+        "sw_versions": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Software Versions",
+          "description": "The software version(s) of the certified module."
+        },
+        "mentioned_certs": {
+          "type": [
+            "object",
+            "null"
+          ],
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_str_id"
+              },
+              "uniqueItems": true
+            }
+          },
+          "title": "Mentioned Certificates",
+          "description": "Other FIPS certificates mentioned in the module page."
+        },
+        "historical_reason": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Historical Reason",
+          "description": "The reason why the certificate is marked as historical (if any)."
+        },
+        "date_sunset": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "format": "date",
+          "title": "Sunset Date",
+          "description": "The date when the certificate was sunsetted/revoked/archived (YYYY-MM-DD)."
+        },
+        "revoked_reason": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "title": "Revocation Reason",
+          "description": "The reason why the certificate was revoked (if any)."
+        },
+        "revoked_link": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "format": "uri",
+          "title": "Revocation Link",
+          "description": "A link to more information about the revocation (if any)."
+        }
+      }
+    },
+    "pdf_data": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.PdfData"
+        },
+        "keywords": {
+          "$ref": "base.json#/definitions/document_keywords",
+          "title": "Security Policy Keywords",
+          "description": "Keywords extracted from the security policy document."
+        },
+        "policy_metadata": {
+          "$ref": "base.json#/definitions/document_metadata",
+          "title": "Security Policy Metadata",
+          "description": "Metadata extracted from the security policy document."
+        }
+      },
+      "title": "Extracted PDF Data",
+      "description": "Data extracted from the certification PDF document (security policy)."
+    },
+    "heuristics": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.Heuristics"
+        },
+        "algorithms": {
+          "type": "object",
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          }
+        },
+        "extracted_versions": {
+          "$ref": "base.json#/definitions/extracted_versions"
+        },
+        "cpe_matches": {
+          "$ref": "base.json#/definitions/cpe_matches"
+        },
+        "verified_cpe_matches": {
+          "$ref": "base.json#/definitions/verified_cpe_matches"
+        },
+        "related_cves": {
+          "$ref": "base.json#/definitions/related_cves"
+        },
+        "policy_prunned_references": {
+          "type": "object",
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_str_id"
+              },
+              "uniqueItems": true
+            }
+          }
+        },
+        "module_prunned_references": {
+          "type": "object",
+          "properties": {
+            "_type": {
+              "const": "Set"
+            },
+            "elements": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/fips_str_id"
+              },
+              "uniqueItems": true
+            }
+          }
+        },
+        "policy_processed_references": {
+          "$ref": "#/definitions/fips_references",
+          "description": "References to and from other certificates, based on the security policy document."
+        },
+        "module_processed_references": {
+          "$ref": "#/definitions/fips_references",
+          "description": "References to and from other certificates, based on the module page."
+        },
+        "direct_transitive_cves": {
+          "$ref": "base.json#/definitions/direct_transitive_cves"
+        },
+        "indirect_transitive_cves": {
+          "$ref": "base.json#/definitions/indirect_transitive_cves"
+        }
+      },
+      "additionalProperties": false,
+      "title": "Certificate Heuristics",
+      "description": "Heuristic data extracted from the certification documents."
+    },
+    "state": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.sample.fips.FIPSCertificate.InternalState"
+        },
+        "module_download_ok": {
+          "type": "boolean"
+        },
+        "policy_download_ok": {
+          "type": "boolean"
+        },
+        "policy_convert_garbage": {
+          "type": "boolean"
+        },
+        "policy_convert_ok": {
+          "type": "boolean"
+        },
+        "module_extract_ok": {
+          "type": "boolean"
+        },
+        "policy_extract_ok": {
+          "type": "boolean"
+        },
+        "policy_pdf_hash": {
+          "type": "string",
+          "pattern": "^[0-9a-fA-F]{64}$",
+          "description": "SHA256 hash of the policy PDF file."
+        },
+        "policy_txt_hash": {
+          "type": "string",
+          "pattern": "^[0-9a-fA-F]{64}$",
+          "description": "SHA256 hash of the policy txt file."
+        }
+      },
+      "additionalProperties": false,
+      "title": "Certificate Document States",
+      "description": "State of the document processing pipeline for policy and module documents."
+    }
+  },
+  "additionalProperties": false
+}
diff --git a/src/sec_certs/serialization/schemas/fips_dataset.json b/src/sec_certs/serialization/schemas/fips_dataset.json
new file mode 100644
index 00000000..f9a5caa6
--- /dev/null
+++ b/src/sec_certs/serialization/schemas/fips_dataset.json
@@ -0,0 +1,62 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://sec-certs.org/schemas/fips_dataset.json",
+  "title": "FIPS 140 Dataset",
+  "description": "Schema for a FIPS 140 dataset.",
+  "type": "object",
+  "properties": {
+    "_type": {
+      "const": "sec_certs.dataset.fips.FIPSDataset"
+    },
+    "state": {
+      "type": "object",
+      "properties": {
+        "_type": {
+          "const": "sec_certs.dataset.dataset.Dataset.DatasetInternalState"
+        },
+        "meta_sources_parsed": {
+          "type": "boolean"
+        },
+        "artifacts_downloaded": {
+          "type": "boolean"
+        },
+        "pdfs_converted": {
+          "type": "boolean"
+        },
+        "auxiliary_datasets_processed": {
+          "type": "boolean"
+        },
+        "certs_analyzed": {
+          "type": "boolean"
+        }
+      },
+      "title": "Dataset internal state"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "title": "Timestamp of the dataset creation"
+    },
+    "name": {
+      "type": "string",
+      "title": "Name of the dataset"
+    },
+    "description": {
+      "type": "string",
+      "title": "Description of the dataset"
+    },
+    "n_certs": {
+      "type": "integer",
+      "minimum": 0,
+      "title": "Number of certificates in the dataset"
+    },
+    "certs": {
+      "type": "array",
+      "items": {
+        "$ref": "fips_certificate.json#"
+      },
+      "title": "List of certificates in the dataset"
+    }
+  },
+  "additionalProperties": false
+}