Add MIP and IUT notebook for FIPS.

1 files changed, 372 insertions, 0 deletions

diff --git a/notebooks/fips/in_process.ipynb b/notebooks/fips/in_process.ipynb
new file mode 100644
index 00000000..f22c478e
--- /dev/null
+++ b/notebooks/fips/in_process.ipynb
@@ -0,0 +1,372 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d7d7d8c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sec_certs.dataset.fips_mip import MIPDataset\n",
+    "from sec_certs.dataset.fips_iut import IUTDataset\n",
+    "from sec_certs.sample.fips_mip import MIPStatus\n",
+    "import pandas as pd\n",
+    "import seaborn as sns\n",
+    "import matplotlib.pyplot as plt\n",
+    "import math\n",
+    "import numpy as np\n",
+    "import tqdm\n",
+    "import matplotlib.ticker as mtick\n",
+    "import warnings\n",
+    "from pathlib import Path\n",
+    "\n",
+    "plt.style.use(\"seaborn-whitegrid\")\n",
+    "sns.set_palette(\"deep\")\n",
+    "sns.set_context(\"notebook\") # Set to \"paper\" for use in paper :)\n",
+    "\n",
+    "warnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bd5f0fd6",
+   "metadata": {},
+   "source": [
+    "## IUT dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "80f04ffe",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "iut_dset = IUTDataset.from_web_latest()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "47a4d967",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "iut_global_df = pd.DataFrame(iut_dset.snapshots, columns=[\"timestamp\", \"displayed\", \"not_displayed\", \"total\"])\n",
+    "iut_global_df.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "776323d7",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "iut_melt = iut_global_df.melt(\"timestamp\", var_name=\"type\", value_name=\"count\")\n",
+    "line = sns.lineplot(data=iut_melt, x=\"timestamp\", y=\"count\", hue=\"type\")\n",
+    "line.set(xlabel=\"Date\", ylabel=\"Number of entries\", title=\"Entries in IUT list over time\")\n",
+    "line.legend(title=\"Type\", bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d1626ad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "s0 = iut_dset.snapshots[0]\n",
+    "print(s0.__dict__.keys())\n",
+    "print(next(iter(s0.entries)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "97b726d6",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "def iut_key(entry):\n",
+    "    return entry.module_name, entry.vendor_name, entry.standard\n",
+    "\n",
+    "iut_first_seen = {}\n",
+    "iut_last_seen = {}\n",
+    "for snapshot in sorted(iut_dset.snapshots, key=lambda x: x.timestamp):\n",
+    "    snapshot_date = snapshot.timestamp.date()\n",
+    "    for entry in snapshot.entries:\n",
+    "        entry_key = entry #iut_key(entry) # or entry here\n",
+    "        if entry_key not in iut_first_seen:\n",
+    "            iut_first_seen[entry_key] = snapshot_date\n",
+    "        if entry_key not in iut_last_seen or iut_last_seen[entry_key] < snapshot_date:\n",
+    "            iut_last_seen[entry_key] = snapshot_date\n",
+    "\n",
+    "#iut_local_df = pd.DataFrame([(entry[0], entry[1], entry[2], iut_first_seen[entry], iut_last_seen[entry], iut_last_seen[entry] == snapshot_date) for entry in iut_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"first_seen\", \"last_seen\", \"present\"))\n",
+    "iut_local_df = pd.DataFrame([(entry.module_name, entry.vendor_name, entry.standard, entry.iut_date, iut_first_seen[entry], iut_last_seen[entry], iut_last_seen[entry] == snapshot_date) for entry in iut_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"iut_date\", \"first_seen\", \"last_seen\", \"present\"))\n",
+    "iut_local_df = iut_local_df.astype({\"standard\": \"category\", \"iut_date\": \"datetime64\", \"first_seen\": \"datetime64\", \"last_seen\": \"datetime64\", \"present\": \"bool\"}).fillna(value=np.nan)\n",
+    "iut_local_df[\"seen_for_iut\"] = (iut_local_df.last_seen - iut_local_df.iut_date).dt.days\n",
+    "iut_local_df[\"seen_for\"] = (iut_local_df.last_seen - iut_local_df.first_seen).dt.days\n",
+    "\n",
+    "print(\"All:\")\n",
+    "print(\"Average seen for\", np.mean(iut_local_df.seen_for))\n",
+    "print(\"Average seen for (FIPS 140-2)\", np.mean(iut_local_df.loc[iut_local_df.standard == \"FIPS 140-2\"].seen_for))\n",
+    "print(\"Average seen for (FIPS 140-3)\", np.mean(iut_local_df.loc[iut_local_df.standard == \"FIPS 140-3\"].seen_for))\n",
+    "print(\"Average seen for[iut]\", np.mean(iut_local_df.seen_for_iut))\n",
+    "print(\"Average seen for[iut] (FIPS 140-2)\", np.mean(iut_local_df.loc[iut_local_df.standard == \"FIPS 140-2\"].seen_for_iut))\n",
+    "print(\"Average seen for[iut] (FIPS 140-3)\", np.mean(iut_local_df.loc[iut_local_df.standard == \"FIPS 140-3\"].seen_for_iut))\n",
+    "\n",
+    "print(\"Only not present:\")\n",
+    "print(\"Average seen for\", np.mean(iut_local_df.loc[~iut_local_df.present].seen_for))\n",
+    "print(\"Average seen for (FIPS 140-2)\", np.mean(iut_local_df.loc[(iut_local_df.standard == \"FIPS 140-2\") & ~iut_local_df.present].seen_for))\n",
+    "print(\"Average seen for (FIPS 140-3)\", np.mean(iut_local_df.loc[(iut_local_df.standard == \"FIPS 140-3\") & ~iut_local_df.present].seen_for))\n",
+    "print(\"Average seen for[iut]\", np.mean(iut_local_df.loc[~iut_local_df.present].seen_for_iut))\n",
+    "print(\"Average seen for[iut] (FIPS 140-2)\", np.mean(iut_local_df.loc[(iut_local_df.standard == \"FIPS 140-2\") & ~iut_local_df.present].seen_for_iut))\n",
+    "print(\"Average seen for[iut] (FIPS 140-3)\", np.mean(iut_local_df.loc[(iut_local_df.standard == \"FIPS 140-3\") & ~iut_local_df.present].seen_for_iut))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5bcc6da5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scatter = sns.relplot(kind=\"scatter\", data=iut_local_df, x=\"iut_date\", y=\"seen_for_iut\", hue=\"standard\", style=\"present\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10bad193",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scatter = sns.relplot(kind=\"scatter\", data=iut_local_df, x=\"iut_date\", y=\"last_seen\", hue=\"standard\", style=\"present\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fbfbf88e",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "hist = sns.histplot(data=iut_local_df[~iut_local_df.present], x=\"seen_for_iut\", hue=\"standard\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98b27134",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hist = sns.histplot(data=iut_local_df[~iut_local_df.present], x=\"seen_for\", hue=\"standard\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6b8e02d3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "iut_local_df.vendor.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cbc02977",
+   "metadata": {},
+   "source": [
+    "## MIP dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46ce3c13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mip_dset = MIPDataset.from_web_latest()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa74b965",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mip_global_df = pd.DataFrame(mip_dset.snapshots, columns=[\"timestamp\", \"displayed\", \"not_displayed\", \"total\"])\n",
+    "mip_global_df.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc3186f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mip_melt = mip_global_df.melt(\"timestamp\", var_name=\"type\", value_name=\"count\")\n",
+    "line = sns.lineplot(data=mip_melt, x=\"timestamp\", y=\"count\", hue=\"type\")\n",
+    "line.set(xlabel=\"Date\", ylabel=\"Number of entries\", title=\"Entries in MIP list over time\")\n",
+    "line.legend(title=\"Type\", bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f433f0c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "m0 = mip_dset.snapshots[0]\n",
+    "print(m0.__dict__.keys())\n",
+    "print(next(iter(m0.entries)))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "205b9a97",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def mip_key(entry):\n",
+    "    return entry.module_name, entry.vendor_name, entry.standard, entry.status\n",
+    "\n",
+    "mip_first_seen = {}\n",
+    "mip_last_seen = {}\n",
+    "for snapshot in sorted(mip_dset.snapshots, key=lambda x: x.timestamp):\n",
+    "    snapshot_date = snapshot.timestamp.date()\n",
+    "    for entry in snapshot.entries:\n",
+    "        entry_key = entry # mip_key(entry) # or entry here\n",
+    "        if entry_key not in mip_first_seen:\n",
+    "            mip_first_seen[entry_key] = snapshot_date\n",
+    "        if entry_key not in mip_last_seen or mip_last_seen[entry_key] < snapshot_date:\n",
+    "            mip_last_seen[entry_key] = snapshot_date\n",
+    "\n",
+    "#mip_local_df = pd.DataFrame([(entry[0], entry[1], entry[2], entry[3], mip_first_seen[entry], mip_last_seen[entry], mip_last_seen[entry] == snapshot_date) for entry in iut_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"status\", first_seen\", \"last_seen\", \"present\"))\n",
+    "mip_local_df = pd.DataFrame([(entry.module_name, entry.vendor_name, entry.standard, entry.status, entry.status_since, mip_first_seen[entry], mip_last_seen[entry], mip_last_seen[entry] == snapshot_date) for entry in mip_first_seen.keys()], columns=(\"name\", \"vendor\", \"standard\", \"status\", \"status_since\", \"first_seen\", \"last_seen\", \"present\"))\n",
+    "mip_local_df = mip_local_df.astype({\"standard\": \"category\", \"status\": \"category\", \"status_since\": \"datetime64\", \"first_seen\": \"datetime64\", \"last_seen\": \"datetime64\", \"present\": \"bool\"}).fillna(value=np.nan)\n",
+    "mip_local_df[\"seen_for_status\"] = (mip_local_df.last_seen - mip_local_df.status_since).dt.days\n",
+    "mip_local_df[\"seen_for\"] = (mip_local_df.last_seen - mip_local_df.first_seen).dt.days\n",
+    "mip_local_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29679c73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scatter = sns.relplot(kind=\"scatter\", data=mip_local_df.loc[mip_local_df.status == MIPStatus.REVIEW_PENDING], x=\"first_seen\", y=\"seen_for\", hue=\"standard\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "771662ff",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scatter = sns.relplot(kind=\"scatter\", data=mip_local_df.loc[mip_local_df.status == MIPStatus.IN_REVIEW], x=\"first_seen\", y=\"seen_for\", hue=\"standard\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2b2ba494",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scatter = sns.relplot(kind=\"scatter\", data=mip_local_df.loc[mip_local_df.status == MIPStatus.COORDINATION], x=\"first_seen\", y=\"seen_for\", hue=\"standard\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "883667b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scatter = sns.relplot(kind=\"scatter\", data=mip_local_df.loc[mip_local_df.status == MIPStatus.FINALIZATION], x=\"first_seen\", y=\"seen_for\", hue=\"standard\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9856e568",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for status in MIPStatus:\n",
+    "    print(status)\n",
+    "    print(\"All:\")\n",
+    "    print(\"Average seen for\", np.mean(mip_local_df.loc[mip_local_df.status == status].seen_for))\n",
+    "    print(\"Average seen for (FIPS 140-2)\", np.mean(mip_local_df.loc[(mip_local_df.status == status) & (mip_local_df.standard == \"FIPS 140-2\")].seen_for))\n",
+    "    print(\"Average seen for (FIPS 140-3)\", np.mean(mip_local_df.loc[(mip_local_df.status == status) & (mip_local_df.standard == \"FIPS 140-3\")].seen_for))\n",
+    "    \n",
+    "    print(\"Only not present:\")\n",
+    "    print(\"Average seen for\", np.mean(mip_local_df.loc[~(mip_local_df.present) & (mip_local_df.status == status)].seen_for))\n",
+    "    print(\"Average seen for (FIPS 140-2)\", np.mean(mip_local_df.loc[~(mip_local_df.present) & (mip_local_df.status == status) & (mip_local_df.standard == \"FIPS 140-2\")].seen_for))\n",
+    "    print(\"Average seen for (FIPS 140-3)\", np.mean(mip_local_df.loc[~(mip_local_df.present) & (mip_local_df.status == status) & (mip_local_df.standard == \"FIPS 140-3\")].seen_for))\n",
+    "    print()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ccf7e2fc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with sns.plotting_context(\"notebook\", font_scale=0.75):\n",
+    "    g = sns.FacetGrid(mip_local_df.loc[~mip_local_df.present], col=\"status\", hue=\"standard\", col_wrap=2, height=2, ylim=(0,300))\n",
+    "    g.map(sns.histplot, \"seen_for\")\n",
+    "    g.set_titles(\"{col_name}\")\n",
+    "    plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2593c5cf",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}