Merge pull request #342 from crocs-muni/feat/new-latex-commands-vulnerability-notebook

new LaTeX commands vuln. notebook

1 files changed, 52 insertions, 18 deletions

diff --git a/notebooks/cc/vulnerabilities.ipynb b/notebooks/cc/vulnerabilities.ipynb
index e6422759..315f35a8 100644
--- a/notebooks/cc/vulnerabilities.ipynb
+++ b/notebooks/cc/vulnerabilities.ipynb
@@ -1,6 +1,7 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -52,6 +53,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -65,24 +67,36 @@
    "outputs": [],
    "source": [
     "# Local instantiation\n",
-    "# dset: CCDataset = CCDataset.from_json(\"/path/to/cc_dataset.json\")\n",
-    "# #dset.process_maintenance_updates() # Run this only once, can take ~10 minutes to finnish, fully processes mainten#ance updates\n",
-    "# main_dset = CCDatasetMaintenanceUpdates.from_json(dset.mu_dataset_path) # TODO: Recover me\n",
-    "# cve_dset: CVEDataset = dset._prepare_cve_dataset()\n",
-    "# cpe_dset: CPEDataset = dset._prepare_cpe_dataset()\n",
     "\n",
-    "# Remote instantiation\n",
-    "with tempfile.TemporaryDirectory() as tmp_dir:\n",
-    "    dset: CCDataset = CCDataset.from_web_latest()\n",
-    "    dset.root_dir = tmp_dir\n",
-    "    main_dset: CCDatasetMaintenanceUpdates = CCDatasetMaintenanceUpdates.from_web_latest()\n",
-    "    dset._prepare_cpe_dataset()\n",
-    "    dset._prepare_cve_dataset()\n",
-    "    cve_dset = dset.auxiliary_datasets.cve_dset\n",
-    "    cpe_dset = dset.auxiliary_datasets.cpe_dset"
+    "print(\"Loading CCDataset\")\n",
+    "dset: CCDataset = CCDataset.from_json(\"/path/to/cc_dset.json\")\n",
+    "\n",
+    "print(\"Loading CCDatasetMaintenanceUpdates\")\n",
+    "main_dset: CCDatasetMaintenanceUpdates = CCDatasetMaintenanceUpdates.from_json(\"/path/to/maintenance_updates.json\")\n",
+    "\n",
+    "print(\"Loading CVEDataset\")\n",
+    "cve_dset: CVEDataset = CVEDataset.from_json(\"/path/to/cve_dataset.json\")\n",
+    "\n",
+    "print(\"Loading CPEDataset\")\n",
+    "cpe_dset: CPEDataset = CPEDataset.from_json(\"/path/to/cpe_dataset.json\")\n",
+    "\n",
+    "# # Remote instantiation (takes approx. 10 minutes to complete)\n",
+    "# with tempfile.TemporaryDirectory() as tmp_dir:\n",
+    "#     dset: CCDataset = CCDataset.from_web_latest()\n",
+    "#     dset.root_dir = tmp_dir\n",
+    "\n",
+    "#     print(\"Downloading dataset of maintenance updates\")\n",
+    "#     main_dset: CCDatasetMaintenanceUpdates = CCDatasetMaintenanceUpdates.from_web_latest()\n",
+    "\n",
+    "#     print(\"Downloading CPE dataset\")\n",
+    "#     cpe_dset: CPEDataset = dset._prepare_cpe_dataset()\n",
+    "\n",
+    "#     print(\"Downloading CVE dataset\")\n",
+    "#     cve_dset: CVEDataset = dset._prepare_cve_dataset()"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -120,8 +134,10 @@
     "\n",
     "df = expand_df_with_cve_cols(df, cve_dset)\n",
     "\n",
+    "df = df.loc[(df.not_valid_before.notnull()) & (df.not_valid_after.notnull())]\n",
+    "\n",
     "df_cves_within_validity_period = filter_to_cves_within_validity_period(\n",
-    "    df.loc[(df.not_valid_before.notnull()) & (df.not_valid_after.notnull())].copy(), cve_dset\n",
+    "    df.copy(), cve_dset\n",
     ")\n",
     "df_cves_within_validity_period = expand_df_with_cve_cols(df_cves_within_validity_period, cve_dset)\n",
     "\n",
@@ -142,6 +158,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -229,6 +246,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -241,7 +259,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# TODO: The number of evaluated families is suspiciously low. Must investigate before recomputing results.\n",
     "# This limits analysis to SAR families that are popular-enough (>100 CVE-rich certs, second-most-popular value with >= 40 instances)\n",
     "cve_rich = df_cves_within_validity_period.loc[df_cves_within_validity_period.related_cves.notnull()].copy()\n",
     "families = discover_sar_families(cve_rich.extracted_sars)\n",
@@ -289,6 +306,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -332,7 +350,7 @@
     "    df_main_after_vuln,\n",
     "    main_dset.to_pandas(),\n",
     "    updates_that_should_fix_vulns_path,\n",
-    "    \"/Users/adam/phd/projects/certificates/sec-certs/datasets/cc_04_10_2022/certs/maintenances/reports/pdf\",\n",
+    "    \"/Users/adam/phd/projects/certificates/sec-certs/datasets/cc_final_run_may_23/auxiliary_datasets/maintenances/reports/pdf\",\n",
     ")\n"
    ]
   },
@@ -370,6 +388,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -441,6 +460,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -484,6 +504,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -529,6 +550,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -559,6 +581,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -594,6 +617,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -678,6 +702,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -720,6 +745,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -814,9 +840,17 @@
     "print(f\"\\\\newcommand{{\\\\numCertsRevokedWithinYear}}{{${df_w_validity_dates.shape[0]}$}}\")\n",
     "print(f\"\\\\newcommand{{\\\\numVulnerableCertsRevokedWithinYear}}{{${maybe_revoked.shape[0]}$}}\")\n",
     "print(\n",
+    "    f\"\\\\newcommand{{\\\\numVulnerableCertsBeforeCertification}}{{${(100 * ratio_before_cert):.0f}\\%$}}\"\n",
+    ")\n",
+    "print(\n",
+    "    f\"\\\\newcommand{{\\\\numVulnerableCertsAfterCertification}}{{${(100 * ratio_after_cert):.0f}\\%$}}\"\n",
+    ")\n",
+    "print(\n",
     "    f\"\\\\newcommand{{\\\\numVulnerableCertsInValidityPeriod}}{{${(100 * n_cves_in_validity_period / n_cves_all):.0f}\\%$}}\"\n",
     ")\n",
     "\n",
+    "\n",
+    "\n",
     "name_mapping = {\n",
     "    \"Improper Restriction of Operations within the Bounds of a Memory Buffer\": \"Buffer overflow\",\n",
     "    \"Exposure of Sensitive Information to an Unauthorized Actor\": \"Sensitive information exposure\",\n",
@@ -861,7 +895,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.13"
+   "version": "3.8.16"
   },
   "vscode": {
    "interpreter": {