Move notebooks around and fix imports.

2 files changed, 1 insertions, 460 deletions

diff --git a/data/cert_id_eval/cert_id_eval.ipynb b/data/cert_id_eval/cert_id_eval.ipynb
deleted file mode 100644
index 1c1b369c..00000000
--- a/data/cert_id_eval/cert_id_eval.ipynb
+++ /dev/null
@@ -1,459 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "collapsed": true,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "# CC certificate id evaluation\n",
-    "This notebook can be used to evaluate our heuristics for certificate id assignment\n",
-    "and canonicalization.\n",
-    "\n",
-    "It looks at several aspects & issues:\n",
-    "\n",
-    "1. Certificates with no id assigned.\n",
-    "2. Duplicate certificate id assignments (when two certificates get the same ID assigned).\n",
-    "3. Certificates that have the same certification report document (an issue of the input data that we get\n",
-    "   that explains some of the duplicate certificate id assignments)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "from sec_certs.dataset import CCDataset\n",
-    "import csv"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "# dset = CCDataset.from_web_latest()\n",
-    "dset = CCDataset.from_json(\"../../cc_dset/CommonCriteria_dataset.json\")\n",
-    "# dset._compute_normalized_cert_ids()\n",
-    "# dset.to_json()"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Certificates with no id\n",
-    "\n",
-    "Here we report the number of certificates in our dataset that we have no certificate ID for."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "missing_id_dgsts = set()\n",
-    "for cert in dset:\n",
-    "    if not cert.heuristics.cert_id:\n",
-    "        print(cert.dgst, cert.heuristics.cert_id, cert.scheme)\n",
-    "        missing_id_dgsts.add(cert.dgst)\n",
-    "print(f\"Total: {len(missing_id_dgsts)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Check manually evaluated missing\n"
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "i = 0\n",
-    "with open(\"missing_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        try:\n",
-    "            cert = dset[line[\"id\"]]\n",
-    "        except:\n",
-    "            continue\n",
-    "        if line[\"cert_id\"] and line[\"cert_id\"] != cert.heuristics.cert_id:\n",
-    "            i += 1\n",
-    "            print(line[\"id\"], line[\"cert_id\"], cert.heuristics.cert_id, line[\"reason\"])\n",
-    "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "\n",
-    "The following cell checks which manually analyzed missing certificate IDs were since fixed."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "manual_missing_ids = set()\n",
-    "i = 0\n",
-    "with open(\"missing_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        manual_missing_ids.add(line[\"id\"])\n",
-    "        if line[\"id\"] not in missing_id_dgsts:\n",
-    "            i += 1\n",
-    "            print(\",\".join(line.values()))\n",
-    "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "The following cell lists missing certificate IDs that *went missing* since manual analysis."
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "new_missing_ids = missing_id_dgsts.difference(manual_missing_ids)\n",
-    "for idd in new_missing_ids:\n",
-    "    cert = dset[idd]\n",
-    "    print(idd, cert.heuristics.cert_id)\n",
-    "print(f\"Total: {len(new_missing_ids)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Duplicate certificate id assignment\n",
-    "\n",
-    "Here we report the number of certificates in our dataset that have a duplicate certiticate\n",
-    "ID assigned."
-   ],
-   "metadata": {
-    "collapsed": false
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "id_mapping = {}\n",
-    "for cert in dset:\n",
-    "    if cert.heuristics.cert_id is not None:\n",
-    "        c_list = id_mapping.setdefault(cert.heuristics.cert_id, [])\n",
-    "        c_list.append(cert.dgst)\n",
-    "\n",
-    "duplicate_id_dgsts = set()\n",
-    "for idd, entries in id_mapping.items():\n",
-    "    if len(entries) > 1 and idd:\n",
-    "        print(idd, entries)\n",
-    "        duplicate_id_dgsts.update(entries)\n",
-    "print(f\"Total: {len(duplicate_id_dgsts)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "## Duplicate report documents\n",
-    "\n",
-    "Some certificates have erroneously uploaded certificate reports, here we check their\n",
-    "hashes and report such duplicates in the input data."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "duplicate_docs = {}\n",
-    "\n",
-    "for cert in dset:\n",
-    "    if cert.state.report_pdf_hash is not None:\n",
-    "        r_list = duplicate_docs.setdefault(cert.state.report_pdf_hash, [])\n",
-    "        r_list.append(cert.dgst)\n",
-    "\n",
-    "duplicate_doc_dgsts = set()\n",
-    "for hash, entries in duplicate_docs.items():\n",
-    "    if len(entries) > 1:\n",
-    "        print(hash, entries)\n",
-    "        for entry in entries:\n",
-    "            duplicate_doc_dgsts.add(entry)\n",
-    "print(f\"Total: {len(duplicate_doc_dgsts)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "The following prints the amount of certificate id duplicates due to input data (two or more\n",
-    "certificates share a certification report document)."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n",
-    "print(len(duplicate_ids_due_doc))"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "The following prints the amount of certificate id duplicates that are not due to input data."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n",
-    "print(len(duplicate_ids_issue))"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "### Check manually evaluated duplicates\n",
-    "\n",
-    "The following cell checks that id collisions that were manually analyzed in the past have been fixed.\n",
-    "A `True` means that we have now assigned the correct ID."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "i = 0\n",
-    "with open(\"duplicate_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        try:\n",
-    "            cert = dset[line[\"id\"]]\n",
-    "        except:\n",
-    "            continue\n",
-    "        if line[\"true_id\"] != cert.heuristics.cert_id:\n",
-    "            print(line[\"id\"],line[\"result\"], line[\"true_id\"] == cert.heuristics.cert_id , line[\"true_id\"], cert.heuristics.cert_id, line[\"fixable\"])\n",
-    "            i += 1\n",
-    "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "The following cell lists those duplicates that were fixed by changes since manual analysis."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "manual_duplicate_ids = set()\n",
-    "i = 0\n",
-    "with open(\"duplicate_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        manual_duplicate_ids.add(line[\"id\"])\n",
-    "        if line[\"id\"] not in duplicate_id_dgsts:\n",
-    "            i += 1\n",
-    "            print(\",\".join(line.values()))\n",
-    "print(f\"Total: {i}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "markdown",
-   "source": [
-    "The following cell lists duplicates that were *created* since manual analysis."
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [
-    "new_duplicate_ids = duplicate_id_dgsts.difference(manual_duplicate_ids)\n",
-    "for idd in new_duplicate_ids:\n",
-    "    cert = dset[idd]\n",
-    "    print(idd, cert.heuristics.cert_id)\n",
-    "print(f\"Total: {len(new_duplicate_ids)}\")"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "outputs": [],
-   "source": [],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
\ No newline at end of file
diff --git a/data/cert_id_eval/missing_ids.csv b/data/cert_id_eval/missing_ids.csv
index 40645785..f7296cbc 100644
--- a/data/cert_id_eval/missing_ids.csv
+++ b/data/cert_id_eval/missing_ids.csv
@@ -42,7 +42,7 @@ ace7ea6d7f58dbb1,DE,,404 for report PDF
 8ed77a6223a94fb5,US,,maybe "09-1832-R-0027 V1.0" the ETR id
 422dc5758723c7d1,TR,21.0.03/TSE-CCCS-68,report not a PDF (docx)
 02482228eb547c15,US,,404 for report PDF
-4de4a9f436958574,CA,55 LSS 2020,report not a PDF (docx)
+4de4a9f436958574,CA,525 LSS 2020,report not a PDF (docx)
 c312bbf9e303b844,SE,CSEC2019012,space in cert_id
 fb6d7faf0ed2dca5,CA,,cert_id missing
 b3ae94eeccc566ba,CA,383-4-17-CR,regex missing