bump some analysis

1 files changed, 168 insertions, 76 deletions

diff --git a/notebooks/cc/references.ipynb b/notebooks/cc/references.ipynb
index 6ce7d137..74e50dc5 100644
--- a/notebooks/cc/references.ipynb
+++ b/notebooks/cc/references.ipynb
@@ -25,9 +25,10 @@
    },
    "outputs": [],
    "source": [
+    "import itertools\n",
     "import warnings\n",
+    "from collections.abc import Iterable\n",
     "from pathlib import Path\n",
-    "from typing import Iterable\n",
     "\n",
     "import matplotlib.pyplot as plt\n",
     "import networkx as nx\n",
@@ -69,9 +70,9 @@
     "plt.rcParams[\"savefig.pad_inches\"] = 0.01\n",
     "sns.set_palette(\"deep\")\n",
     "\n",
-    "# plt.style.use(\"seaborn-whitegrid\")\n",
-    "# sns.set_palette(\"deep\")\n",
-    "# sns.set_context(\"notebook\")  # Set to \"paper\" for use in paper :)\n",
+    "plt.style.use(\"default\")\n",
+    "sns.set_palette(\"deep\")\n",
+    "sns.set_context(\"notebook\")  # Set to \"paper\" for use in paper :)\n",
     "\n",
     "# plt.rcParams['figure.figsize'] = (10, 6)\n",
     "\n",
@@ -125,9 +126,10 @@
     "            }\n",
     "        )\n",
     "        .assign(\n",
-    "            longer_than_5_years=lambda df_: df_.not_valid_after - df_.not_valid_before > pd.Timedelta(days=5 * 365),\n",
+    "            longer_than_5_years=lambda df_: df_.not_valid_after - df_.not_valid_before\n",
+    "            > pd.Timedelta(days=(5 * 365) + 2),\n",
     "            not_valid_after=lambda df_: df_.not_valid_after.where(\n",
-    "                ~df_.longer_than_5_years, df_.not_valid_before + pd.Timedelta(days=5 * 365)\n",
+    "                ~df_.longer_than_5_years, df_.not_valid_before + pd.Timedelta(days=(5 * 365) + 2)\n",
     "            ),\n",
     "        )\n",
     "        .drop_duplicates(subset=[\"cert_id\"], keep=\"first\")  # TODO: Investigate high number of duplicates and resolve\n",
@@ -191,9 +193,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dset = CCDataset.from_json(\"/Users/adam/phd/projects/certificates/sec-certs/dataset/cc_final_run_may_23/dataset.json\")\n",
+    "dset = CCDataset.from_json(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/cc_november_23/dataset.json\")\n",
     "cc_df = preprocess_cc_df(dset.to_pandas())\n",
-    "refs_df = preprocess_refs_df(\"/Users/adam/Downloads/predictions.csv\", cc_df)\n",
+    "refs_df = preprocess_refs_df(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/reference_prediction/predictions.csv\", cc_df)\n",
     "unique_labels = refs_df.reference_label.unique().tolist()\n",
     "\n",
     "# Load labeled reference graph as networkx directed graph\n",
@@ -205,7 +207,10 @@
     "    create_using=nx.DiGraph,\n",
     ")\n",
     "\n",
-    "cc_df = compute_reference_numbers(compute_references(cc_df, graph, unique_labels))\n"
+    "cc_df = compute_reference_numbers(compute_references(cc_df, graph, unique_labels))\n",
+    "cc_df_comp = compute_reference_numbers(compute_references(cc_df, graph, \"COMPONENT_USED\"))\n",
+    "cc_df_prev = compute_reference_numbers(compute_references(cc_df, graph, \"PREVIOUS_VERSION\"))\n",
+    "assert cc_df.n_refs.sum() == cc_df_comp.n_refs.sum() + cc_df_prev.n_refs.sum()\n"
    ]
   },
   {
@@ -216,20 +221,6 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Understand which columns I need and limit myself to those columns\n",
-    "# Every analytical cell should be isolated in a function that takes a single input: The dataframe of certificates to work on.\n",
-    "#     - The number of those references is computed in the function itself\n",
-    "#     - Each analytical method should have some tests at the end\n",
-    "#     - If some LaTeX output accompanies the computaiton, the function should return it as a string\n",
-    "#     - Those are stored in a dictionary that keeps expanding\n"
-   ]
-  },
-  {
    "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
@@ -243,20 +234,37 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def compute_basic_reference_graph_stats(df__: pd.DataFrame, graph: nx.DiGraph) -> dict[str, str]:\n",
-    "    df = df__.copy().assign(has_refs=lambda df_: df_.refs.notnull()).pipe(compute_reference_numbers)\n",
+    "def compute_basic_reference_graph_stats(\n",
+    "    df__: pd.DataFrame, df_comp__: pd.DataFrame, df_prev__: pd.DataFrame\n",
+    ") -> dict[str, str]:\n",
+    "    df = df__.copy().assign(has_refs=lambda df_: df_.refs.notnull())\n",
+    "    df_comp = df_comp__.copy().assign(has_refs=lambda df_: df_.refs.notnull())\n",
+    "    df_prev = df_prev__.copy().assign(has_refs=lambda df_: df_.refs.notnull())\n",
     "\n",
     "    n_ref_smartcards = df.loc[(df.has_refs) & (df.category == SMARTCARD_CATEGORY)].shape[0]\n",
     "    n_ref_others = df.loc[(df.has_refs) & (df.category != SMARTCARD_CATEGORY)].shape[0]\n",
     "\n",
+    "    n_comp_smartcards = df_comp.loc[(df_comp.category == SMARTCARD_CATEGORY) & (df_comp.has_refs)].shape[0]\n",
+    "    n_comp_others = df_comp.loc[(df_comp.category != SMARTCARD_CATEGORY) & (df_comp.has_refs)].shape[0]\n",
+    "\n",
+    "    n_prev_smartcards = df_prev.loc[(df_prev.category == SMARTCARD_CATEGORY) & (df_prev.has_refs)].shape[0]\n",
+    "    n_prev_others = df_prev.loc[(df_prev.category != SMARTCARD_CATEGORY) & (df_prev.has_refs)].shape[0]\n",
+    "\n",
+    "    print(\n",
+    "        f\"Total number of referencing certificates: {n_ref_smartcards + n_ref_others} ({100 * (n_ref_smartcards + n_ref_others) / df.shape[0]:.2f}%)\"\n",
+    "    )\n",
     "    print(\n",
     "        f\"Number of smartcard certificates that reference some other certificate: {n_ref_smartcards} ({100 * n_ref_smartcards / df.loc[df.category == SMARTCARD_CATEGORY].shape[0]:.2f}%)\"\n",
     "    )\n",
+    "    print(f\"\\t- Out of that, {n_comp_smartcards} do reference a sub-component.\")\n",
+    "    print(f\"\\t- Out of that, {n_prev_smartcards} do reference a previous version.\")\n",
     "    print(\n",
     "        f\"Number of non-smartcard certificates that reference some other certificate: {n_ref_others} ({100 * n_ref_others / df.loc[df.category != SMARTCARD_CATEGORY].shape[0]:.2f}%)\"\n",
     "    )\n",
+    "    print(f\"\\t- Out of that, {n_comp_others} do reference a sub-component.\")\n",
+    "    print(f\"\\t- Out of that, {n_prev_others} do reference a previous version.\")\n",
     "    print(\n",
-    "        f\"Total number of referencing certificates: {n_ref_smartcards + n_ref_others} ({100 * (n_ref_smartcards + n_ref_others) / df.shape[0]:.2f}%)\"\n",
+    "        \"Note that the numbers don't sum to 100, as a certificate can reference both a sub-component and a previous version.\"\n",
     "    )\n",
     "\n",
     "    df_melted = df[[\"n_refs\", \"n_trans_refs\", \"n_in_refs\", \"n_in_trans_refs\"]].melt()\n",
@@ -269,7 +277,7 @@
     "    return {}\n",
     "\n",
     "\n",
-    "compute_basic_reference_graph_stats(cc_df, graph)\n"
+    "compute_basic_reference_graph_stats(cc_df, cc_df_comp, cc_df_prev)\n"
    ]
   },
   {
@@ -286,11 +294,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# TODO: Check that it actually works, the data on small subset was fairly weird\n",
-    "# TODO: Work only on sub-component references?\n",
-    "def compute_certs_top_reach(df__: pd.DataFrame) -> dict:\n",
+    "# TODO: Decide on and implement the note below\n",
+    "\"\"\"\n",
+    "This implementation is naive. When computing on full graph, RoCA vulnerable certs. are the most popular. They are re-evaluated bunch of times. On top of that, each iteration is referendec by\n",
+    "a decent number of certificates in a sub-compoennt relationship.\n",
+    "The product is not a single certificate, but rather a chain of n-times re-evaluated certificates. In order to capture the true number of sub-component references, I'd need to sum the references in all the iterations (for each re-evaluation).\n",
+    "\"\"\"\n",
+    "\n",
+    "\n",
+    "def compute_certs_top_reach(df__: pd.DataFrame):\n",
     "    def find_reach_over_time(df_: pd.DataFrame, cert_id: str, date_range: pd.DatetimeIndex) -> pd.Series:\n",
-    "        df = df_.copy().loc[lambda df_: df_.in_trans_refs.apply(lambda x: pd.notnull(x) and cert_id in x)]\n",
+    "        df = df_.copy().loc[lambda df_: df_.trans_refs.apply(lambda x: pd.notnull(x) and cert_id in x)]\n",
     "        dct = {\n",
     "            date: df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)].shape[0] for date in date_range\n",
     "        }\n",
@@ -299,6 +313,8 @@
     "    df = df__.copy()\n",
     "    top_10_certs = df.sort_values(by=\"n_in_trans_refs\", ascending=False).head(10)\n",
     "    print(top_10_certs[[\"cert_id\", \"n_in_trans_refs\"]])\n",
+    "    for dgst in top_10_certs.index.tolist():\n",
+    "        print(dset[dgst].name)\n",
     "\n",
     "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
     "    data = [find_reach_over_time(df, x, date_range) for x in tqdm(top_10_certs.cert_id.tolist())]\n",
@@ -314,10 +330,10 @@
     "    plt.savefig(RESULTS_DIR / \"lineplot_top_certificate_reach.pdf\", bbox_inches=\"tight\")\n",
     "    plt.show()\n",
     "\n",
-    "    return {}\n",
+    "    return top_10_certs.index.tolist()\n",
     "\n",
     "\n",
-    "compute_certs_top_reach(cc_df)\n"
+    "top_10_digests = compute_certs_top_reach(cc_df_comp)\n"
    ]
   },
   {
@@ -406,8 +422,17 @@
     "    return {}\n",
     "\n",
     "\n",
-    "compute_avg_references_over_time(cc_df)\n",
-    "compute_avg_reach_over_time(cc_df)\n"
+    "compute_avg_references_over_time(cc_df_comp)\n",
+    "compute_avg_reach_over_time(cc_df_comp)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cc_df_comp.head()\n"
    ]
   },
   {
@@ -533,8 +558,8 @@
     "    return {}\n",
     "\n",
     "\n",
-    "compute_number_of_active_vs_ref_rich_certs_over_time(cc_df)\n",
-    "compute_summary_active_vs_ref_rich_over_time(cc_df)\n"
+    "compute_number_of_active_vs_ref_rich_certs_over_time(cc_df_comp)\n",
+    "compute_summary_active_vs_ref_rich_over_time(cc_df_comp)\n"
    ]
   },
   {
@@ -562,24 +587,26 @@
     "\n",
     "    for date in tqdm(date_range):\n",
     "        active_certs = df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)].copy()\n",
-    "        active_certs_cert_ids = set(active_certs[\"cert_id\"].tolist())\n",
-    "        active_certs[\"no_intersection\"] = active_certs.refs.map(\n",
-    "            lambda x: False if pd.isnull(x) else not x.intersection(active_certs_cert_ids)\n",
+    "        archived_certs = df.loc[(date < df.not_valid_before) | (date > df.not_valid_after)].copy()\n",
+    "        archived_cert_ids = set(archived_certs[\"cert_id\"].tolist())\n",
+    "        active_certs[\"ref_archived\"] = active_certs.refs.map(\n",
+    "            lambda x: False if pd.isnull(x) else bool(x.intersection(archived_cert_ids))\n",
     "        )\n",
-    "        active_certs[\"no_transitive_intersection\"] = active_certs.trans_refs.map(\n",
-    "            lambda x: False if pd.isnull(x) else not x.intersection(active_certs_cert_ids)\n",
+    "        active_certs[\"trans_ref_archived\"] = active_certs.trans_refs.map(\n",
+    "            lambda x: False if pd.isnull(x) else bool(x.intersection(archived_cert_ids))\n",
     "        )\n",
+    "\n",
     "        dct_direct_others[date] = active_certs.loc[\n",
-    "            (active_certs.no_intersection) & (active_certs.category != SMARTCARD_CATEGORY)\n",
+    "            (active_certs.ref_archived) & (active_certs.category != SMARTCARD_CATEGORY)\n",
     "        ].shape[0]\n",
     "        dct_transitive_others[date] = active_certs.loc[\n",
-    "            (active_certs.no_transitive_intersection) & (active_certs.category != SMARTCARD_CATEGORY)\n",
+    "            (active_certs.trans_ref_archived) & (active_certs.category != SMARTCARD_CATEGORY)\n",
     "        ].shape[0]\n",
     "        dct_direct_smartcards[date] = active_certs.loc[\n",
-    "            (active_certs.no_intersection) & (active_certs.category == SMARTCARD_CATEGORY)\n",
+    "            (active_certs.ref_archived) & (active_certs.category == SMARTCARD_CATEGORY)\n",
     "        ].shape[0]\n",
     "        dct_transitive_smartcards[date] = active_certs.loc[\n",
-    "            (active_certs.no_transitive_intersection) & (active_certs.category == SMARTCARD_CATEGORY)\n",
+    "            (active_certs.trans_ref_archived) & (active_certs.category == SMARTCARD_CATEGORY)\n",
     "        ].shape[0]\n",
     "\n",
     "    df_refs_to_archived_melted = (\n",
@@ -609,7 +636,7 @@
     "    return {}\n",
     "\n",
     "\n",
-    "compute_certs_referencing_archived_ones(cc_df)\n"
+    "compute_certs_referencing_archived_ones(cc_df_comp)\n"
    ]
   },
   {
@@ -682,7 +709,7 @@
     "    return {}\n",
     "\n",
     "\n",
-    "compute_certs_referencing_vulnerable_over_time(cc_df)\n"
+    "compute_certs_referencing_vulnerable_over_time(cc_df_comp)\n"
    ]
   },
   {
@@ -714,7 +741,7 @@
     "        countplot = sns.countplot(data=df, x=\"category\", hue=col, ax=axes[index])\n",
     "        countplot.set(\n",
     "            xlabel=\"Category\",\n",
-    "            ylabel=\"Outgoing direct references\",\n",
+    "            ylabel=f\"{' '.join(col.split('_'))}\",\n",
     "            title=f\"Countplot of {' '.join(col.split('_'))}\",\n",
     "        )\n",
     "        countplot.tick_params(axis=\"x\", rotation=90)\n",
@@ -725,7 +752,8 @@
     "    return {}\n",
     "\n",
     "\n",
-    "plot_direct_refs_per_category(cc_df)\n"
+    "plot_direct_refs_per_category(cc_df_comp)\n",
+    "plot_direct_refs_per_category(cc_df_prev)\n"
    ]
   },
   {
@@ -777,7 +805,7 @@
     "    return {}\n",
     "\n",
     "\n",
-    "plot_sankey_refs_categories(cc_df)\n"
+    "plot_sankey_refs_categories(cc_df_comp)\n"
    ]
   },
   {
@@ -834,26 +862,25 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "def countplot_certs_referencing_archived(df__: pd.DataFrame) -> dict:\n",
-    "    def references_archived_cert(references):\n",
-    "        if pd.isnull(references):\n",
-    "            return False\n",
-    "\n",
-    "        return any([x in cert_ids] for x in references)\n",
-    "\n",
+    "def compute_certs_referencing_archived_ones(df__: pd.DataFrame) -> dict:\n",
     "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
+    "    certs_that_reference_archived_one = set()\n",
+    "\n",
+    "    for date in tqdm(date_range):\n",
+    "        active_certs = df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)].copy()\n",
+    "        archived_certs = df.loc[(date < df.not_valid_before) | (date > df.not_valid_after)].copy()\n",
+    "        archived_cert_ids = set(archived_certs[\"cert_id\"].tolist())\n",
+    "        active_certs[\"refs_archived\"] = active_certs.refs.map(\n",
+    "            lambda x: False if pd.isnull(x) else bool(x.intersection(archived_cert_ids))\n",
+    "        )\n",
+    "        certs_that_reference_archived_one.update(active_certs.loc[(active_certs.refs_archived)].index.tolist())\n",
     "\n",
-    "    cert_ids = set(df.loc[((df.cert_id.notnull()) & (df.status == \"archived\")), \"cert_id\"].tolist())\n",
-    "    df[\"references_archived_cert\"] = df.in_refs.map(references_archived_cert)\n",
+    "    df[\"references_archived_cert\"] = df.index.map(lambda x: x in certs_that_reference_archived_one)\n",
     "\n",
-    "    # TODO: We should limit on the number of certificates that referenced an archived certificate at some point where they were active as well.\n",
     "    print(\n",
     "        f\"Number of certificates that reference some archived certificate: {df.loc[df.references_archived_cert].shape[0]}\"\n",
     "    )\n",
@@ -879,7 +906,7 @@
     "    return {}\n",
     "\n",
     "\n",
-    "countplot_certs_referencing_archived(cc_df)\n"
+    "compute_certs_referencing_archived_ones(cc_df_comp)\n"
    ]
   },
   {
@@ -935,11 +962,35 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Reference network visualization"
+    "## Assurance mismatch\n",
+    "\n",
+    "- Search for certificates that use stronger EAL than the references subcomponent\n",
+    "- Examine cycles in the subcomponent graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TBA\n",
+    "# Create a function that takes two arguments: cc_dset, eal, and references, and returns whether the cert. references something with lower EAL\n",
+    "# Just compute this boolean column for all certs\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Re-evaluation popularity\n",
+    "\n",
+    "- How many certificates do undergo re-evaluation?\n",
+    "- How many rounds of re-evaluation do certificates undergo?\n",
+    "- How often (timewise) do these certificates undergo re-evaluation?"
    ]
   },
   {
@@ -948,11 +999,52 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Print:\n",
-    "# - How many references in reports\n",
-    "# - How many references in targets\n",
-    "# - How many references in total\n",
-    "# -\n"
+    "def plot_ref_label_popularity_over_time(df__: pd.DataFrame, cc_df_comp__: pd.DataFrame, cc_df_prev__: pd.DataFrame):\n",
+    "    df = df__.copy()\n",
+    "    date_range = pd.date_range(df.not_valid_before.min(), df.not_valid_before.max())\n",
+    "    sub_comp_refs = {}\n",
+    "    prev_refs = {}\n",
+    "    active_certs_dct = {}\n",
+    "    for date in tqdm(date_range):\n",
+    "        active_sub_comp_certs = cc_df_comp__.loc[\n",
+    "            (date >= cc_df_comp__.not_valid_before) & (date <= cc_df_comp__.not_valid_after)\n",
+    "        ].copy()\n",
+    "        active_prev_certs = cc_df_prev__.loc[\n",
+    "            (date >= cc_df_prev__.not_valid_before) & (date <= cc_df_prev__.not_valid_after)\n",
+    "        ].copy()\n",
+    "        sub_comp_refs[date] = active_sub_comp_certs.n_refs.sum()\n",
+    "        prev_refs[date] = active_prev_certs.n_refs.sum()\n",
+    "        active_certs_dct[date] = df.loc[(date >= df.not_valid_before) & (date <= df.not_valid_after)].shape[0]\n",
+    "\n",
+    "    df_references_melted = (\n",
+    "        pd.concat(\n",
+    "            [\n",
+    "                pd.Series(sub_comp_refs, name=\"sub-component\"),\n",
+    "                pd.Series(prev_refs, name=\"previous_version\"),\n",
+    "                pd.Series(active_certs_dct, name=\"active certificates\"),\n",
+    "            ],\n",
+    "            axis=1,\n",
+    "        )\n",
+    "        .rename_axis(\"date\")\n",
+    "        .reset_index()\n",
+    "        .melt(id_vars=[\"date\"], var_name=\"reference type\", value_name=\"number of certificates\")\n",
+    "    )\n",
+    "\n",
+    "    g = sns.lineplot(data=df_references_melted, x=\"date\", y=\"number of certificates\", hue=\"reference type\")\n",
+    "    g.set(\n",
+    "        title=\"Sum of references in currently active certs. with different labels in time\",\n",
+    "        xlabel=\"Time\",\n",
+    "        ylabel=\"Number of references\",\n",
+    "    )\n",
+    "    plt.savefig(RESULTS_DIR / \"lineplot_different_labels.pdf\", bbox_inches=\"tight\")\n",
+    "    plt.show()\n",
+    "\n",
+    "\n",
+    "plot_ref_label_popularity_over_time(cc_df, cc_df_comp, cc_df_prev)\n",
+    "\n",
+    "print(\n",
+    "    f\"Number of certificates that did undergo re-evaluation (or previous version): {len(set(itertools.chain.from_iterable(cc_df_prev.loc[cc_df_prev.refs.notnull()].refs.tolist())))}\"\n",
+    ")\n"
    ]
   },
   {
@@ -1007,7 +1099,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "nx.draw(view, pos=nx.planar_layout(view), with_labels=True)\n"
+    "nx.draw(view, with_labels=True)\n"
    ]
   },
   {
@@ -1123,7 +1215,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.6"
   },
   "vscode": {
    "interpreter": {