path: root/notebooks/fips/temporal_trends.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Temporal trends in the FIPS-140 ecosystem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "from sec_certs.dataset import FIPSDataset\n",
    "from sec_certs.utils.extract import rules_get_subset, extract_key_paths\n",
    "from sec_certs.cert_rules import fips_rules, PANDAS_KEYWORDS_CATEGORIES\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import numpy as np\n",
    "import tqdm\n",
    "import matplotlib.ticker as mtick\n",
    "import warnings\n",
    "from pathlib import Path\n",
    "\n",
    "plt.style.use(\"seaborn-whitegrid\")\n",
    "sns.set_palette(\"deep\")\n",
    "sns.set(rc={\"figure.figsize\":(8, 4)})\n",
    "sns.set_context(\"notebook\") # Set to \"paper\" for use in paper :)\n",
    "\n",
    "warnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "dset = FIPSDataset.from_web()\n",
    "df = dset.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "categories = df.loc[df.year_from < 2022].groupby([\"year_from\", \"type\"], as_index=False).size().copy()\n",
    "\n",
    "with sns.plotting_context(\"notebook\", font_scale=0.75):\n",
    "    g = sns.FacetGrid(categories, col=\"type\", hue=\"type\", col_wrap=3, height=4, ylim=(0,190))\n",
    "    g.map(sns.lineplot, \"year_from\", \"size\")\n",
    "    g.set(xlabel=\"Year of cert.\", ylabel=\"N. certs.\")\n",
    "    g.set_titles(\"{col_name}\")\n",
    "    g.fig.subplots_adjust(top=0.90)\n",
    "    g.fig.suptitle('Module type prevalence in time')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "embodiments = df.loc[(df.embodiment.notnull()) & (df.year_from < 2022)].groupby([\"year_from\", \"embodiment\"], as_index=False, observed=True).size()\n",
    "\n",
    "line = sns.lineplot(data=embodiments, x=\"year_from\", y=\"size\", hue=\"embodiment\")\n",
    "line.set(xlabel=\"Year of certification\", ylabel=\"Number of issued certificates\", title=\"Embodiment prevalence in time\")\n",
    "line.legend(title=\"Embodiment\", bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "standards = df.loc[(df.standard.notnull()) & (df.year_from < 2022)].groupby([\"year_from\", \"standard\"], as_index=False, observed=True).size()\n",
    "\n",
    "line = sns.lineplot(data=standards, x=\"year_from\", y=\"size\", hue=\"standard\")\n",
    "line.set(xlabel=\"Year of certification\", ylabel=\"Number of issued certificates\", title=\"Standard prevalence in time\")\n",
    "line.legend(title=\"Standard\", bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "sec_levels = df.loc[(df.level.notnull()) & (df.year_from < 2022)].groupby([\"year_from\", \"level\"], as_index=False).size()\n",
    "sec_levels.level = sec_levels.level.astype(int)\n",
    "\n",
    "line = sns.lineplot(data=sec_levels, x=\"year_from\", y=\"size\", hue=\"level\")\n",
    "line.set(xlabel=\"Year of certification\", ylabel=\"Number of issued certificates\", title=\"Security level prevalence in time\")\n",
    "line.legend(title=\"Level\", bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "avg_levels = df.loc[(df.year_from < 2022) & (df.level.notnull())].copy().groupby([\"year_from\"]).agg({'year_from':'size', 'level':'mean'}).rename(columns={\"year_from\": \"n_certs\"}).reset_index()\n",
    "avg_levels.year_from = avg_levels.year_from.astype(\"float\")\n",
    "avg_levels.level = avg_levels.level.astype(\"float\")\n",
    "\n",
    "ymin = math.floor(avg_levels.level.min())\n",
    "ymax = math.ceil(avg_levels.level.max())\n",
    "\n",
    "line = sns.lineplot(data=avg_levels, x=\"year_from\", y=\"level\", marker='o')\n",
    "line.set(xlabel=\"Year of certification\", ylabel=\"Average security level\", title=\"Average security level over time\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "avg_levels = df.loc[(df.year_from < 2022) & (df.level.notnull())].copy().groupby([\"year_from\", \"type\"]).agg({'year_from':'size', 'level':'mean'}).rename(columns={\"year_from\": \"n_certs\"}).reset_index()\n",
    "avg_levels.year_from = avg_levels.year_from.astype(\"float\")\n",
    "avg_levels.level = avg_levels.level.astype(\"float\")\n",
    "\n",
    "with sns.plotting_context(\"notebook\", font_scale=0.75):\n",
    "    g = sns.FacetGrid(avg_levels, col=\"type\", hue=\"type\", col_wrap=3, height=4)\n",
    "    g.map(sns.lineplot, \"year_from\", \"level\")\n",
    "    g.set(xlabel=\"Year of cert.\", ylabel=\"Avg. level\")\n",
    "    g.set_titles(\"{col_name}\")\n",
    "    g.fig.subplots_adjust(top=0.90)\n",
    "    g.fig.suptitle('Average security level between types')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "df[\"validity_period\"] = (df.date_sunset - df.date_validation).dt.days / 365\n",
    "\n",
    "df_validity = df.loc[(df.validity_period.notnull()) & (df.year_from < 2022)]\n",
    "validity_period = df_validity.groupby(\"year_from\").agg({'year_from':'size', 'validity_period':'mean'}).rename(columns={\"year_from\": \"n_certs\"}).reset_index()\n",
    "\n",
    "figure, axes = plt.subplots(1, 2)\n",
    "figure.set_size_inches(12, 4)\n",
    "figure.set_tight_layout(True)\n",
    "\n",
    "line = sns.lineplot(data=validity_period, x=\"year_from\", y=\"validity_period\", marker=\"o\", ax=axes[0])\n",
    "line.set(xlabel=\"Year of certification\", ylabel=\"Average lifetime of certificates (in years)\", title=\"Average lifetime of certificates in years\")\n",
    "\n",
    "box = sns.boxplot(data=df_validity, x=\"year_from\", y=\"validity_period\", ax=axes[1])\n",
    "box.set(xlabel=\"Year of certification\", ylabel=\"Lifetime of certificates (in years)\", title=\"Boxplot of certificate validity periods in individual years\")\n",
    "box.tick_params(axis='x', rotation=60)\n",
    "\n",
    "strips = sns.relplot(kind=\"scatter\", data=df_validity, x=\"date_validation\", y=\"validity_period\", height=10, aspect=2/1, hue=\"type\")\n",
    "strips.set(title=\"Scatter plot of validity period development over time\", xlabel=\"Date of certification\", ylabel=\"Validity period of certificate (in years)\")\n",
    "\n",
    "scatter = sns.relplot(kind=\"scatter\", data=df_validity, x=\"date_validation\", y=\"date_sunset\", height=10, aspect=2/1, hue=\"type\")\n",
    "scatter.set(title=\"Scatter plot of validity dates\", xlabel=\"Date of certification (not valid before)\", ylabel=\"Date of expiry (not valid after)\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# TODO: Resolve duplicity in crypto_scheme mac\n",
    "st_keywords_df = dset.get_keywords_df(\"keywords\").drop(columns=[\"crypto_scheme.MAC\"])\n",
    "st_keywords_df = st_keywords_df.rename(columns={x: x.split(\".\")[-1] for x in st_keywords_df.columns}).fillna(0).applymap(lambda x: x > 0)\n",
    "\n",
    "df_keywords = df.loc[:, [\"type\", \"level\", \"date_validation\", \"date_sunset\", \"year_from\"]].copy()\n",
    "df_keywords = df_keywords.join(st_keywords_df).loc[df_keywords.year_from < 2022].copy()\n",
    "\n",
    "figure, axes = plt.subplots(4, 3)\n",
    "figure.set_size_inches(20, 20)\n",
    "figure.set_tight_layout(True)\n",
    "row = 0\n",
    "col = 0\n",
    "\n",
    "for examined_category in PANDAS_KEYWORDS_CATEGORIES:\n",
    "    rules_subset = rules_get_subset(examined_category)\n",
    "    keywords = [x.split(\".\")[-1] for x in extract_key_paths(rules_subset, examined_category)]\n",
    "    top_n_keywords = df_keywords.loc[:, keywords].sum().sort_values(ascending=False).head(10).index\n",
    "\n",
    "    # Count number of non-zero rows for each year, weight by number of certificates issued in the given year.\n",
    "    crypto = df_keywords.groupby(\"year_from\")[top_n_keywords].sum()\n",
    "    crypto[\"n_certs\"] = df_keywords.groupby(\"year_from\").size()\n",
    "    crypto.iloc[:,:-1] = crypto.iloc[:,:-1].div(crypto.n_certs, axis=0) * 100\n",
    "    crypto = crypto.drop(columns=[\"n_certs\"]).reset_index().melt(id_vars=\"year_from\", var_name=\"keyword\", value_name=\"percentage\")  # Bring to tidy form\n",
    "\n",
    "    line = sns.lineplot(data=crypto, x=\"year_from\", y=\"percentage\", hue=\"keyword\", ax=axes[row][col])\n",
    "    line.set(title=f\"Density of {examined_category} keywords over time\", xlabel=\"Year of certification\", ylabel=\"% of certs. containing keyword\")\n",
    "    line.yaxis.set_major_formatter(mtick.PercentFormatter())\n",
    "    line.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n",
    "\n",
    "    if col == 2:\n",
    "        col = 0\n",
    "        row += 1\n",
    "    else:\n",
    "        col += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# TODO: This needs refactoring. Currently we don't hold FIPSAlgorithm objects, just strings\n",
    "# def algo_present(algorithms, algo):\n",
    "#     for a in algorithms:\n",
    "#         if a.algorithm_type == algo:\n",
    "#             return True\n",
    "#     return False\n",
    "\n",
    "# algo_types = Counter()\n",
    "# for algos in df.algorithms:\n",
    "#     for algo in algos:\n",
    "#         if algo.algorithm_type:\n",
    "#             algo_types[algo.algorithm_type] +=1\n",
    "# #print(algo_types)\n",
    "# df_algos = df.loc[:, [\"type\", \"level\", \"date_validation\", \"date_sunset\", \"year_from\", \"algorithms\"]].copy()\n",
    "# for algo, count in algo_types.most_common(14):\n",
    "#     df_algos[algo] = df_algos.algorithms.apply(algo_present, args=(algo,))\n",
    "\n",
    "# crypto = df_algos.groupby(\"year_from\").sum()\n",
    "# crypto[\"n_certs\"] = df_algos.groupby(\"year_from\").size()\n",
    "# crypto.iloc[:,:-1] = crypto.iloc[:,:-1].div(crypto.n_certs, axis=0) * 100\n",
    "# crypto = crypto.drop(columns=[\"level\",\"n_certs\"]).reset_index().melt(id_vars=\"year_from\", var_name=\"keyword\", value_name=\"percentage\")  # Bring to tidy form\n",
    "\n",
    "# line = sns.lineplot(data=crypto, x=\"year_from\", y=\"percentage\", hue=\"keyword\")\n",
    "# line.set(title=f\"Density of algorithm types over time\", xlabel=\"Year of certification\", ylabel=\"% of certs. containing algorithm type\")\n",
    "# line.yaxis.set_major_formatter(mtick.PercentFormatter())\n",
    "# line.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3.8.13 ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "a5b8c5b127d2cfe5bc3a1c933e197485eb9eba25154c3661362401503b4ef9d4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}