Move notebooks around and fix imports.

6 files changed, 152 insertions, 205 deletions

diff --git a/data/cert_id_eval/missing_ids.csv b/data/cert_id_eval/missing_ids.csv
index 40645785..f7296cbc 100644
--- a/data/cert_id_eval/missing_ids.csv
+++ b/data/cert_id_eval/missing_ids.csv
@@ -42,7 +42,7 @@ ace7ea6d7f58dbb1,DE,,404 for report PDF
 8ed77a6223a94fb5,US,,maybe "09-1832-R-0027 V1.0" the ETR id
 422dc5758723c7d1,TR,21.0.03/TSE-CCCS-68,report not a PDF (docx)
 02482228eb547c15,US,,404 for report PDF
-4de4a9f436958574,CA,55 LSS 2020,report not a PDF (docx)
+4de4a9f436958574,CA,525 LSS 2020,report not a PDF (docx)
 c312bbf9e303b844,SE,CSEC2019012,space in cert_id
 fb6d7faf0ed2dca5,CA,,cert_id missing
 b3ae94eeccc566ba,CA,383-4-17-CR,regex missing
diff --git a/data/readme.md b/data/readme.md
index 37d1f8e1..6fe44e65 100644
--- a/data/readme.md
+++ b/data/readme.md
@@ -1,5 +1,14 @@
 # Data
 
+## Certificate ID evaluation
+
+The directory `./cert_id_eval` contains data on a manual evaluation of certificate ID assignment.
+`missing_ids.csv` contains an evaluation of certificates to which the sec-certs tool was not able to
+find a certificate ID (to analyze why that happened and whether we could fix that).
+`duplicate_ids.csv` contains an evaluation of certificates to which the sec-certs tool assigned a duplicate
+ID (to analyze why that happened and whether we could fix that). These files are used by the `cert_id_eval.ipynb`
+Jupyter notebook which evaluates a dataset with respect to the manually labeled ground truth in them.
+
 ## CPE Matching
 
 To evaluate quality of our automatic matching of CPE names to certificates (which in turn allows us to assign CVEs to certificates), we manually evaluated candidate CPE records for all of the certificates both in FIPS and CC framework. 
diff --git a/notebooks/cc/references.ipynb b/notebooks/cc/references.ipynb
index 2dce9909..29084c61 100644
--- a/notebooks/cc/references.ipynb
+++ b/notebooks/cc/references.ipynb
@@ -191,9 +191,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.10.7"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 1
-}
+}
\ No newline at end of file
diff --git a/data/cert_id_eval/cert_id_eval.ipynb b/notebooks/cert_id_eval.ipynb
index 1c1b369c..c0463750 100644
--- a/data/cert_id_eval/cert_id_eval.ipynb
+++ b/notebooks/cert_id_eval.ipynb
@@ -24,52 +24,55 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "from sec_certs.dataset import CCDataset\n",
-    "import csv"
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "from sec_certs.dataset import CCDataset\n",
+    "import csv"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "# dset = CCDataset.from_web_latest()\n",
-    "dset = CCDataset.from_json(\"../../cc_dset/CommonCriteria_dataset.json\")\n",
+    "dset = CCDataset.from_json(\"../cc_dset/CommonCriteria_dataset.json\")\n",
     "# dset._compute_normalized_cert_ids()\n",
     "# dset.to_json()"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "## Certificates with no id\n",
-    "\n",
-    "Here we report the number of certificates in our dataset that we have no certificate ID for."
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "## Certificates with no id\n",
+    "\n",
+    "Here we report the number of certificates in our dataset that we have no certificate ID for."
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    },
+    "scrolled": false
+   },
    "outputs": [],
    "source": [
     "missing_id_dgsts = set()\n",
@@ -78,30 +81,27 @@
     "        print(cert.dgst, cert.heuristics.cert_id, cert.scheme)\n",
     "        missing_id_dgsts.add(cert.dgst)\n",
     "print(f\"Total: {len(missing_id_dgsts)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "### Check manually evaluated missing\n"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "i = 0\n",
-    "with open(\"missing_ids.csv\", \"r\") as f:\n",
+    "with open(\"../data/cert_id_eval/missing_ids.csv\", \"r\") as f:\n",
     "    reader = csv.DictReader(f)\n",
     "    for line in reader:\n",
     "        try:\n",
@@ -112,35 +112,33 @@
     "            i += 1\n",
     "            print(line[\"id\"], line[\"cert_id\"], cert.heuristics.cert_id, line[\"reason\"])\n",
     "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "\n",
-    "The following cell checks which manually analyzed missing certificate IDs were since fixed."
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "\n",
+    "The following cell checks which manually analyzed missing certificate IDs were since fixed."
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "manual_missing_ids = set()\n",
     "i = 0\n",
-    "with open(\"missing_ids.csv\", \"r\") as f:\n",
+    "with open(\"../data/cert_id_eval/missing_ids.csv\", \"r\") as f:\n",
     "    reader = csv.DictReader(f)\n",
     "    for line in reader:\n",
     "        manual_missing_ids.add(line[\"id\"])\n",
@@ -148,56 +146,50 @@
     "            i += 1\n",
     "            print(\",\".join(line.values()))\n",
     "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "The following cell lists missing certificate IDs that *went missing* since manual analysis."
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "new_missing_ids = missing_id_dgsts.difference(manual_missing_ids)\n",
     "for idd in new_missing_ids:\n",
     "    cert = dset[idd]\n",
-    "    print(idd, cert.heuristics.cert_id)\n",
+    "    print(idd, cert.heuristics.cert_id, cert.pdf_data.report_filename)\n",
     "print(f\"Total: {len(new_missing_ids)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
+   "metadata": {},
    "source": [
     "## Duplicate certificate id assignment\n",
     "\n",
     "Here we report the number of certificates in our dataset that have a duplicate certiticate\n",
     "ID assigned."
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "id_mapping = {}\n",
@@ -212,32 +204,30 @@
     "        print(idd, entries)\n",
     "        duplicate_id_dgsts.update(entries)\n",
     "print(f\"Total: {len(duplicate_id_dgsts)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
    "source": [
     "## Duplicate report documents\n",
     "\n",
     "Some certificates have erroneously uploaded certificate reports, here we check their\n",
     "hashes and report such duplicates in the input data."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "duplicate_docs = {}\n",
@@ -254,91 +244,85 @@
     "        for entry in entries:\n",
     "            duplicate_doc_dgsts.add(entry)\n",
     "print(f\"Total: {len(duplicate_doc_dgsts)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "The following prints the amount of certificate id duplicates due to input data (two or more\n",
-    "certificates share a certification report document)."
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "The following prints the amount of certificate id duplicates due to input data (two or more\n",
+    "certificates share a certification report document)."
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n",
-    "print(len(duplicate_ids_due_doc))"
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n",
+    "print(len(duplicate_ids_due_doc))"
+   ]
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "The following prints the amount of certificate id duplicates that are not due to input data."
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "The following prints the amount of certificate id duplicates that are not due to input data."
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n",
-    "print(len(duplicate_ids_issue))"
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n",
+    "print(len(duplicate_ids_issue))"
+   ]
   },
   {
    "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
    "source": [
     "### Check manually evaluated duplicates\n",
     "\n",
     "The following cell checks that id collisions that were manually analyzed in the past have been fixed.\n",
     "A `True` means that we have now assigned the correct ID."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "i = 0\n",
-    "with open(\"duplicate_ids.csv\", \"r\") as f:\n",
+    "with open(\"../data/cert_id_eval/duplicate_ids.csv\", \"r\") as f:\n",
     "    reader = csv.DictReader(f)\n",
     "    for line in reader:\n",
     "        try:\n",
@@ -349,34 +333,32 @@
     "            print(line[\"id\"],line[\"result\"], line[\"true_id\"] == cert.heuristics.cert_id , line[\"true_id\"], cert.heuristics.cert_id, line[\"fixable\"])\n",
     "            i += 1\n",
     "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "The following cell lists those duplicates that were fixed by changes since manual analysis."
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "The following cell lists those duplicates that were fixed by changes since manual analysis."
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "manual_duplicate_ids = set()\n",
     "i = 0\n",
-    "with open(\"duplicate_ids.csv\", \"r\") as f:\n",
+    "with open(\"../data/cert_id_eval/duplicate_ids.csv\", \"r\") as f:\n",
     "    reader = csv.DictReader(f)\n",
     "    for line in reader:\n",
     "        manual_duplicate_ids.add(line[\"id\"])\n",
@@ -384,29 +366,27 @@
     "            i += 1\n",
     "            print(\",\".join(line.values()))\n",
     "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "markdown",
-   "source": [
-    "The following cell lists duplicates that were *created* since manual analysis."
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%% md\n"
     }
-   }
+   },
+   "source": [
+    "The following cell lists duplicates that were *created* since manual analysis."
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "new_duplicate_ids = duplicate_id_dgsts.difference(manual_duplicate_ids)\n",
@@ -414,46 +394,35 @@
     "    cert = dset[idd]\n",
     "    print(idd, cert.heuristics.cert_id)\n",
     "print(f\"Total: {len(new_duplicate_ids)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {},
    "outputs": [],
-   "source": [],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
+   "pygments_lexer": "ipython3",
+   "version": "3.10.7"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 1
 }
\ No newline at end of file
diff --git a/notebooks/fips_data.ipynb b/notebooks/fips_data.ipynb
deleted file mode 100644
index 19ffb046..00000000
--- a/notebooks/fips_data.ipynb
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from sec_certs.dataset import FIPSDataset\n",
-    "\n",
-    "dset: FIPSDataset\n",
-    "dset = FIPSDataset.from_web_latest()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Dataset is now ready for exploration. Feel free to explore the data."
-   ]
-  }
- ],
- "metadata": {
-  "language_info": {
-   "name": "python"
-  },
-  "orig_nbformat": 4
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/notebooks/validation_test_split.ipynb b/notebooks/validation_test_split.ipynb
index 18ec26e1..8ed0f690 100644
--- a/notebooks/validation_test_split.ipynb
+++ b/notebooks/validation_test_split.ipynb
@@ -12,7 +12,7 @@
     "\n",
     "```python\n",
     "from sec_certs.model.evaluation import get_validation_dgsts\n",
-    "from sec_cers.dataset import CCDataset\n",
+    "from sec_certs.dataset import CCDataset\n",
     "\n",
     "dset = CCDataset.from_json() # or call FIPSDataset.from_json()\n",
     "validation_dgsts = get_validation_dgsts('/path/to/validation_set.json')\n",
@@ -109,4 +109,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}
\ No newline at end of file