aboutsummaryrefslogtreecommitdiffhomepage
path: root/notebooks
diff options
context:
space:
mode:
Diffstat (limited to 'notebooks')
-rw-r--r--notebooks/cc/references.ipynb4
-rw-r--r--notebooks/cert_id_eval.ipynb428
-rw-r--r--notebooks/fips_data.ipynb31
-rw-r--r--notebooks/validation_test_split.ipynb4
4 files changed, 432 insertions, 35 deletions
diff --git a/notebooks/cc/references.ipynb b/notebooks/cc/references.ipynb
index 2dce9909..29084c61 100644
--- a/notebooks/cc/references.ipynb
+++ b/notebooks/cc/references.ipynb
@@ -191,9 +191,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.4"
+ "version": "3.10.7"
}
},
"nbformat": 4,
"nbformat_minor": 1
-}
+} \ No newline at end of file
diff --git a/notebooks/cert_id_eval.ipynb b/notebooks/cert_id_eval.ipynb
new file mode 100644
index 00000000..c0463750
--- /dev/null
+++ b/notebooks/cert_id_eval.ipynb
@@ -0,0 +1,428 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true,
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "# CC certificate id evaluation\n",
+ "This notebook can be used to evaluate our heuristics for certificate id assignment\n",
+ "and canonicalization.\n",
+ "\n",
+ "It looks at several aspects & issues:\n",
+ "\n",
+ "1. Certificates with no id assigned.\n",
+ "2. Duplicate certificate id assignments (when two certificates get the same ID assigned).\n",
+ "3. Certificates that have the same certification report document (an issue of the input data that we get\n",
+ " that explains some of the duplicate certificate id assignments)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from sec_certs.dataset import CCDataset\n",
+ "import csv"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# dset = CCDataset.from_web_latest()\n",
+ "dset = CCDataset.from_json(\"../cc_dset/CommonCriteria_dataset.json\")\n",
+ "# dset._compute_normalized_cert_ids()\n",
+ "# dset.to_json()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "## Certificates with no id\n",
+ "\n",
+ "Here we report the number of certificates in our dataset that we have no certificate ID for."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ },
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "missing_id_dgsts = set()\n",
+ "for cert in dset:\n",
+ " if not cert.heuristics.cert_id:\n",
+ " print(cert.dgst, cert.heuristics.cert_id, cert.scheme)\n",
+ " missing_id_dgsts.add(cert.dgst)\n",
+ "print(f\"Total: {len(missing_id_dgsts)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Check manually evaluated missing\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "i = 0\n",
+ "with open(\"../data/cert_id_eval/missing_ids.csv\", \"r\") as f:\n",
+ " reader = csv.DictReader(f)\n",
+ " for line in reader:\n",
+ " try:\n",
+ " cert = dset[line[\"id\"]]\n",
+ " except:\n",
+ " continue\n",
+ " if line[\"cert_id\"] and line[\"cert_id\"] != cert.heuristics.cert_id:\n",
+ " i += 1\n",
+ " print(line[\"id\"], line[\"cert_id\"], cert.heuristics.cert_id, line[\"reason\"])\n",
+ "print(f\"Total: {i}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "The following cell checks which manually analyzed missing certificate IDs were since fixed."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "manual_missing_ids = set()\n",
+ "i = 0\n",
+ "with open(\"../data/cert_id_eval/missing_ids.csv\", \"r\") as f:\n",
+ " reader = csv.DictReader(f)\n",
+ " for line in reader:\n",
+ " manual_missing_ids.add(line[\"id\"])\n",
+ " if line[\"id\"] not in missing_id_dgsts:\n",
+ " i += 1\n",
+ " print(\",\".join(line.values()))\n",
+ "print(f\"Total: {i}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The following cell lists missing certificate IDs that *went missing* since manual analysis."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "new_missing_ids = missing_id_dgsts.difference(manual_missing_ids)\n",
+ "for idd in new_missing_ids:\n",
+ " cert = dset[idd]\n",
+ " print(idd, cert.heuristics.cert_id, cert.pdf_data.report_filename)\n",
+ "print(f\"Total: {len(new_missing_ids)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Duplicate certificate id assignment\n",
+ "\n",
+ "Here we report the number of certificates in our dataset that have a duplicate certiticate\n",
+ "ID assigned."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "id_mapping = {}\n",
+ "for cert in dset:\n",
+ " if cert.heuristics.cert_id is not None:\n",
+ " c_list = id_mapping.setdefault(cert.heuristics.cert_id, [])\n",
+ " c_list.append(cert.dgst)\n",
+ "\n",
+ "duplicate_id_dgsts = set()\n",
+ "for idd, entries in id_mapping.items():\n",
+ " if len(entries) > 1 and idd:\n",
+ " print(idd, entries)\n",
+ " duplicate_id_dgsts.update(entries)\n",
+ "print(f\"Total: {len(duplicate_id_dgsts)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "## Duplicate report documents\n",
+ "\n",
+ "Some certificates have erroneously uploaded certificate reports, here we check their\n",
+ "hashes and report such duplicates in the input data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "duplicate_docs = {}\n",
+ "\n",
+ "for cert in dset:\n",
+ " if cert.state.report_pdf_hash is not None:\n",
+ " r_list = duplicate_docs.setdefault(cert.state.report_pdf_hash, [])\n",
+ " r_list.append(cert.dgst)\n",
+ "\n",
+ "duplicate_doc_dgsts = set()\n",
+ "for hash, entries in duplicate_docs.items():\n",
+ " if len(entries) > 1:\n",
+ " print(hash, entries)\n",
+ " for entry in entries:\n",
+ " duplicate_doc_dgsts.add(entry)\n",
+ "print(f\"Total: {len(duplicate_doc_dgsts)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "The following prints the amount of certificate id duplicates due to input data (two or more\n",
+ "certificates share a certification report document)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n",
+ "print(len(duplicate_ids_due_doc))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "The following prints the amount of certificate id duplicates that are not due to input data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n",
+ "print(len(duplicate_ids_issue))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "### Check manually evaluated duplicates\n",
+ "\n",
+ "The following cell checks that id collisions that were manually analyzed in the past have been fixed.\n",
+ "A `True` means that we have now assigned the correct ID."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "i = 0\n",
+ "with open(\"../data/cert_id_eval/duplicate_ids.csv\", \"r\") as f:\n",
+ " reader = csv.DictReader(f)\n",
+ " for line in reader:\n",
+ " try:\n",
+ " cert = dset[line[\"id\"]]\n",
+ " except:\n",
+ " continue\n",
+ " if line[\"true_id\"] != cert.heuristics.cert_id:\n",
+ " print(line[\"id\"],line[\"result\"], line[\"true_id\"] == cert.heuristics.cert_id , line[\"true_id\"], cert.heuristics.cert_id, line[\"fixable\"])\n",
+ " i += 1\n",
+ "print(f\"Total: {i}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "The following cell lists those duplicates that were fixed by changes since manual analysis."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "manual_duplicate_ids = set()\n",
+ "i = 0\n",
+ "with open(\"../data/cert_id_eval/duplicate_ids.csv\", \"r\") as f:\n",
+ " reader = csv.DictReader(f)\n",
+ " for line in reader:\n",
+ " manual_duplicate_ids.add(line[\"id\"])\n",
+ " if line[\"id\"] not in duplicate_id_dgsts:\n",
+ " i += 1\n",
+ " print(\",\".join(line.values()))\n",
+ "print(f\"Total: {i}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "The following cell lists duplicates that were *created* since manual analysis."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "new_duplicate_ids = duplicate_id_dgsts.difference(manual_duplicate_ids)\n",
+ "for idd in new_duplicate_ids:\n",
+ " cert = dset[idd]\n",
+ " print(idd, cert.heuristics.cert_id)\n",
+ "print(f\"Total: {len(new_duplicate_ids)}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+} \ No newline at end of file
diff --git a/notebooks/fips_data.ipynb b/notebooks/fips_data.ipynb
deleted file mode 100644
index 19ffb046..00000000
--- a/notebooks/fips_data.ipynb
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from sec_certs.dataset import FIPSDataset\n",
- "\n",
- "dset: FIPSDataset\n",
- "dset = FIPSDataset.from_web_latest()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Dataset is now ready for exploration. Feel free to explore the data."
- ]
- }
- ],
- "metadata": {
- "language_info": {
- "name": "python"
- },
- "orig_nbformat": 4
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/notebooks/validation_test_split.ipynb b/notebooks/validation_test_split.ipynb
index 18ec26e1..8ed0f690 100644
--- a/notebooks/validation_test_split.ipynb
+++ b/notebooks/validation_test_split.ipynb
@@ -12,7 +12,7 @@
"\n",
"```python\n",
"from sec_certs.model.evaluation import get_validation_dgsts\n",
- "from sec_cers.dataset import CCDataset\n",
+ "from sec_certs.dataset import CCDataset\n",
"\n",
"dset = CCDataset.from_json() # or call FIPSDataset.from_json()\n",
"validation_dgsts = get_validation_dgsts('/path/to/validation_set.json')\n",
@@ -109,4 +109,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
-}
+} \ No newline at end of file
generated by cgit v1.3.1 (git 2.54.0) at 2026-07-21 08:21:38 +0000