Drop dependency_analysis notebook, replaced by References notebook.

1 files changed, 0 insertions, 1168 deletions

diff --git a/notebooks/cc/dependency_analysis.ipynb b/notebooks/cc/dependency_analysis.ipynb
deleted file mode 100644
index 336e264b..00000000
--- a/notebooks/cc/dependency_analysis.ipynb
+++ /dev/null
@@ -1,1168 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "40eb700c",
-   "metadata": {},
-   "source": [
-    "## Dependency analysis\n",
-    "\n",
-    "This notebook contains analysis of dependencies in Common Criteria certificates."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "32456ebd",
-   "metadata": {},
-   "source": [
-    "### Interesting occurences I have noticed during analysing data\n",
-    "- 3970 certificates referencing no other certificates: directly_affecting == Nan && indirectly_affecting == NaN\n",
-    "- 158 certificates which are affected by at least one certificates and affecting no certificates.\n",
-    "- 311 certificates are affected by at least one archived certicates.\n",
-    "- 16 BSI certificates affecting ANSSI certificates out of total 831 BSI certs.\n",
-    "- 38 ANSSI certificates affecting BSI certificates out of total 682 ANSSI certs.\n",
-    "- 25 certificates are crossed referenced\n",
-    "- Certificates with security level EAL6+ are directly affecting other certificates with levels: {'EAL6+': 38, 'EAL5+': 6, 'EAL4+': 5} (EAL6+ is directly affecting certificates with lower security levels)\n",
-    "- Most common security level among smart-cards is EAL5+ with 671 occurences.\n",
-    "- Highest Smart Card BSI level: EAL6+, most common level: EAL4+\n",
-    "- Highest Smart Card ANSSI level: EAL7, most common level: EAL5+\n",
-    "- Lowest security level among smart cards in dataset: EAL1+ ['ATMEL AT90SC6464C Integrated circuit (reference AT568A9 rev. F)',\n",
-    " 'CT2000 embedded Component (reference ST16RFHD50/RSG-A)',\n",
-    " 'M/Chip Select v2.0.5.2 Application',\n",
-    " 'MODEUS electronic purse : MODEUS carrier card v1.1 (reference : ST16RF58/RSE+) and SAM TC/C v1.1 retailer security module (reference : ST19SF16FF/RVN)',\n",
-    " \"Oberthur B0' application v1.0.1 and GemClub v1.3 loaded on Javacard/VOP GemXpresso platform 211 V2\",\n",
-    " 'Palmera Protect platform V2.0 JavaCard (SLE66CX320P/SB62 embedded component)',\n",
-    " 'VOP 2.0.1 / Javacard 2.1.1 JPH33V2 Operating system version 1 installed on Integrated circuit PHILIPS P8WE5033',\n",
-    " 'Javacard/VOP GemXpresso 211 platform (Philips Integrated circuit P8WE5032/MPH02)',\n",
-    " 'Javacard/VOP GemXpresso 211 platform V2 (Philips P8WE5032/MPH04 embedded component, A000000018434D Card Manager)',\n",
-    " 'S3C8975 for smart cards Integrated circuit',\n",
-    " \"'Mondex Purse 2' electronic purse version 0203 component SLE66CX160S, MULTOS V4.1N operating system)\",\n",
-    " \"B4/B0' V2 bank application of the MONEO/CB hybrid card (reference : ST19SF16B RCL version B303/B002)\",\n",
-    " \"Javacard/VOP GemXpresso 211 platform (Philips P8WE5032/MPH02 Integrated circuit ) with Oberthur B0' v0.32 and Visa VSDC v1.08 applets\",\n",
-    " 'MONEO electronic wallet card carrier (ST19SF16B RCL v. B303) and PSAM retailer security module (ST19SF16B RCL v. C103)']\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "36688df6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import seaborn as sns\n",
-    "import matplotlib.pyplot as plt\n",
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "import collections \n",
-    "import datetime\n",
-    "\n",
-    "from sec_certs.dataset import CCDataset\n",
-    "from typing import Tuple, List\n",
-    "\n",
-    "%matplotlib inline\n",
-    "plt.style.use(\"seaborn-whitegrid\")\n",
-    "sns.set_palette(\"deep\")\n",
-    "sns.set_context(\"notebook\")  # Set to \"paper\" for use in paper :)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "257e2aa9",
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "dset: CCDataset = CCDataset.from_web_latest()\n",
-    "dset._compute_dependencies()\n",
-    "df = dset.to_pandas()\n",
-    "\n",
-    "print(f\"Dataset has {df.shape[0]} rows and {df.shape[1]} columns.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e5b76a1a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "98ac89ea",
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "df.info()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "37718d3d",
-   "metadata": {},
-   "source": [
-    "### How many active and archived certificates are in dataset?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a2052bdc",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "df.status.value_counts()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c607ef59",
-   "metadata": {},
-   "source": [
-    "### Which certificates are referenced at least by one certificate? "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1019d39a",
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "def is_directly_affected_by(references):\n",
-    "    if references is np.nan:\n",
-    "        return False\n",
-    "    \n",
-    "    return True\n",
-    "\n",
-    "def count_directly_affected_by(references):\n",
-    "    if references is np.nan:\n",
-    "        return np.nan\n",
-    "    return len(references)\n",
-    "\n",
-    "directly_referencing_df = df.copy()\n",
-    "directly_referencing_df[\"is_directly_referencing\"] = df[\"directly_referencing\"].apply(is_directly_affected_by)\n",
-    "directly_referencing_df[\"directly_referencing_sum\"] = directly_affected_by_df[\"directly_referencing\"].apply(count_directly_affected_by)\n",
-    "directly_referencing_df.sort_values(by=\"directly_referencing_sum\", ascending=False, inplace=True)\n",
-    "directly_referencing_df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ae45f0ef",
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "fig, (ax1, ax2) = plt.subplots(2,1, figsize=(12,20))\n",
-    "normalized_serie = directly_affected_by_df[\"category\"].value_counts(normalize=True)\n",
-    "plt.rcParams.update({'font.size': 20})\n",
-    "sns.countplot(y=\"category\", hue=\"is_directly_referencing\", data=directly_referencing_df, ax=ax1).set_title(\"Which certificates are referenced at least by one certificate vs. which are affected by no certificates\")\n",
-    "sns.barplot(y=normalized_serie.index, x=normalized_serie.values, ax=ax2).set_title(\"Normalized values for each category\")\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "909bfe81",
-   "metadata": {},
-   "source": [
-    "### Which certificates are referencing no other?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a1bf7637",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "no_affecting_df = df[df[\"directly_affecting\"].isna() & df[\"indirectly_affecting\"].isna()]\n",
-    "\n",
-    "print(f\"There are total {no_affecting_df.shape[0]} certificates referencing no other certificates.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "875eba0a",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "no_affecting_df.head()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0fa6bb88",
-   "metadata": {},
-   "source": [
-    "### How many no affecting certificates are affected by other certificates?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "009f9c73",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "affected_but_no_affecting_df = no_affecting_df[no_affecting_df[\"directly_affected_by\"].notna() & no_affecting_df[\"indirectly_affected_by\"].notna()]\n",
-    "print(f\"There are total of {affected_but_no_affecting_df.shape[0]} certificates which are affected by other certificates and affecting no certificates.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0512b68d",
-   "metadata": {},
-   "source": [
-    "### How many certificates are not affected by other certificates, nor affecting other certificates?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f5398edd",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "def is_no_affecting_nor_affected(directly_affecting, indirectly_affecting, directly_affected_by, indirectly_affected_by):\n",
-    "    if directly_affecting is np.nan and indirectly_affecting is np.nan and directly_affected_by is np.nan and indirectly_affected_by is np.nan:\n",
-    "        return True\n",
-    "    \n",
-    "    return False\n",
-    "\n",
-    "\n",
-    "no_affecting_no_affected_df = df.copy()\n",
-    "no_affecting_no_affected_df[\"is_no_affecting_nor_affected\"] = df.apply(lambda x: is_no_affecting_nor_affected(x[\"directly_affecting\"], x[\"indirectly_affecting\"], x[\"directly_affected_by\"], x[\"indirectly_affected_by\"]), axis=1)\n",
-    "no_affecting_no_affected_df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "97870d41",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "fig, (ax1, ax2) = plt.subplots(2,1, figsize=(12,20))\n",
-    "normalized_serie = no_affecting_no_affected_df[\"scheme\"].value_counts(normalize=True)\n",
-    "plt.rcParams.update({'font.size': 20})\n",
-    "sns.countplot(y=\"scheme\", hue=\"is_no_affecting_nor_affected\", data=no_affecting_no_affected_df, ax=ax1).set_title(\"Distribution of schemes which certs from categories are not affecting, nor affected by other certs\")\n",
-    "sns.barplot(y=normalized_serie.index, x=normalized_serie.values, ax=ax2).set_title(\"Normalized values for each scheme\")\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c65fb79b",
-   "metadata": {},
-   "source": [
-    "### Which certificates are dependent on the archived certificates?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fc547955",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "archived_cert_id_list = df[df[\"cert_id\"].notna() & (df[\"status\"] == \"archived\")][\"cert_id\"].tolist()\n",
-    "\n",
-    "def contains_archived_cert_dependency(affected_by):\n",
-    "    if affected_by is np.nan:\n",
-    "        return False\n",
-    "    \n",
-    "    for cert_id in affected_by:\n",
-    "        if cert_id in archived_cert_id_list:\n",
-    "            return True\n",
-    "        \n",
-    "    return False\n",
-    "\n",
-    "\n",
-    "depends_on_archived_df = df.copy()\n",
-    "depends_on_archived_df[\"depends_on_archived\"] = depends_on_archived_df[\"directly_affected_by\"].apply(contains_archived_cert_dependency)\n",
-    "total_records_dependent = sum(depends_on_archived_df[\"depends_on_archived\"])\n",
-    "print(f\"Total {total_records_dependent} certificates are affected by at least one archived certicates.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8a6e379d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "fig, (ax1, ax2) = plt.subplots(2,1, figsize=(12,20))\n",
-    "normalized_serie = depends_on_archived_df[\"category\"].value_counts(normalize=True)\n",
-    "plt.rcParams.update({'font.size': 20})\n",
-    "sns.countplot(y=\"category\", hue=\"depends_on_archived\", data=depends_on_archived_df, ax=ax1).set_title(\"Distribution of categories among certificates dependent on archived certs.\")\n",
-    "sns.barplot(y=normalized_serie.index, x=normalized_serie.values, ax=ax2).set_title(\"Normalized values for each category\")\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "4c7d3d3c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "fig, (ax1, ax2) = plt.subplots(2,1, figsize=(12,20))\n",
-    "normalized_serie = depends_on_archived_df[\"scheme\"].value_counts(normalize=True)\n",
-    "plt.rcParams.update({'font.size': 20})\n",
-    "sns.countplot(y=\"scheme\", hue=\"depends_on_archived\", data=depends_on_archived_df, ax=ax1).set_title(\"Distribution of schemes among certificates dependent on archived certs.\")\n",
-    "sns.barplot(y=normalized_serie.index, x=normalized_serie.values, ax=ax2).set_title(\"Normalized values for each scheme\")\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "75891368",
-   "metadata": {},
-   "source": [
-    "### How frequently are BSI certificates referencing ANSSI certs and vice versa?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e532eb30",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "from typing import Set\n",
-    "\n",
-    "\n",
-    "def is_bsi_cert(cert_id: str) -> bool:\n",
-    "    if cert_id is np.nan:\n",
-    "        return False\n",
-    "    \n",
-    "    if cert_id.lower().startswith(\"bsi\"):\n",
-    "        return True\n",
-    "    \n",
-    "    return False\n",
-    "\n",
-    "\n",
-    "def is_anssi_cert(cert_id: str) -> bool:\n",
-    "    if cert_id is np.nan:\n",
-    "        return False\n",
-    "    \n",
-    "    if cert_id.lower().startswith(\"anssi\"):\n",
-    "        return True\n",
-    "\n",
-    "    return False\n",
-    "\n",
-    "\n",
-    "def is_affecting_anssi(directly_affecting: Set[str]) -> bool:\n",
-    "    if directly_affecting is np.nan:\n",
-    "        return False\n",
-    "    \n",
-    "    for cert_id in directly_affecting:\n",
-    "        if is_anssi_cert(cert_id):\n",
-    "            return True\n",
-    "        \n",
-    "    return False\n",
-    "\n",
-    "\n",
-    "def is_affecting_bsi(directly_affecting: Set[str]) -> bool:\n",
-    "    if directly_affecting is np.nan:\n",
-    "        return False\n",
-    "    \n",
-    "    for cert_id in directly_affecting:\n",
-    "        if is_bsi_cert(cert_id):\n",
-    "            return True\n",
-    "    \n",
-    "    return False\n",
-    "    \n",
-    "    \n",
-    "df[\"is_bsi_cert\"] = df[\"cert_id\"].apply(is_bsi_cert)\n",
-    "df[\"is_anssi_cert\"] = df[\"cert_id\"].apply(is_anssi_cert)\n",
-    "\n",
-    "bsi_df = df[df[\"is_bsi_cert\"] == True].copy()\n",
-    "anssi_df = df[df[\"is_anssi_cert\"] == True].copy()\n",
-    "\n",
-    "bsi_df[\"is_affecting_anssi\"] = bsi_df[\"directly_affecting\"].apply(is_affecting_anssi)\n",
-    "bsi_affecting_anssi_df = bsi_df[bsi_df[\"is_affecting_anssi\"] == True]\n",
-    "\n",
-    "anssi_df[\"is_affecting_bsi\"] = anssi_df[\"directly_affecting\"].apply(is_affecting_bsi)\n",
-    "anssi_affecting_bsi_df = anssi_df[anssi_df[\"is_affecting_bsi\"] == True]\n",
-    "\n",
-    "bsi_total_records = bsi_df.shape[0]\n",
-    "anssi_total_records = anssi_df.shape[0]\n",
-    "bsi_affecting_records = bsi_affecting_anssi_df.shape[0]\n",
-    "anssi_affecting_records = anssi_affecting_bsi_df.shape[0]\n",
-    "\n",
-    "print(f\"There are {bsi_affecting_records} BSI certs affecting ANSSI certs out of total {bsi_total_records} BSI certs.\")\n",
-    "print(f\"There are {anssi_affecting_records} ANSSI certs affecting BSI certs out of total {anssi_total_records} ANSSI certs.\")\n",
-    "\n",
-    "print(f\"Success hit for BSI certs: {bsi_affecting_records / bsi_total_records}\")\n",
-    "print(f\"Success hit for ANSSI certs: {anssi_affecting_records / anssi_total_records}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8613862c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "bsi_affecting_anssi_df.head()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e357818b",
-   "metadata": {},
-   "source": [
-    "### Which certificates are referencing each other? (= are crossed referenced)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1376bb24",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def is_already_involved(cross_reference_list: List[Tuple[str, str]], certs_set: Set[str]) -> bool:\n",
-    "    return certs_set in cross_reference_list\n",
-    "\n",
-    "def is_cert_affecting_other_cert(root_cert_id: str, affected_cert_id: str) -> bool:\n",
-    "    return affected_cert_id in cross_df[cross_df[\"cert_id\"] == root_cert_id].iloc[0][\"directly_affecting\"]\n",
-    "\n",
-    "cross_reference_list: List[Set[str]] = []\n",
-    "cross_df = df[(df[\"cert_id\"].notna()) & (df[\"directly_affecting\"].notna())]\n",
-    "count = 1\n",
-    "total = cross_df.shape[0]\n",
-    "\n",
-    "for cert_record in cross_df.itertuples():\n",
-    "    cert_id = cert_record.cert_id\n",
-    "\n",
-    "    for another_cert_record in cross_df.itertuples():\n",
-    "\n",
-    "        another_cert_id = another_cert_record.cert_id\n",
-    "        \n",
-    "        if cert_record.cert_id == another_cert_record.cert_id:\n",
-    "            continue\n",
-    "            \n",
-    "        certs_set = set([cert_id, another_cert_id])\n",
-    "        \n",
-    "        if is_cert_affecting_other_cert(cert_id, another_cert_id) and is_cert_affecting_other_cert(another_cert_id, cert_id) and not is_already_involved(cross_reference_list, certs_set):\n",
-    "            cross_reference_list.append(certs_set)\n",
-    "    count += 1 \n",
-    "    \n",
-    "    \n",
-    "print(f\"Total of {len(cross_reference_list)} crossed referenced certificates.\")\n",
-    "print(cross_reference_list)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b03da278",
-   "metadata": {},
-   "source": [
-    "### What are the EAL levels typically affecting a certificate? E.g. are certificates referencing EAL5 typically higher or same level?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e604ee18",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c23e0981",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Introduce security level EAL variable\n",
-    "eals = ['EAL1', 'EAL1+', 'EAL2', 'EAL2+', 'EAL3', 'EAL3+', 'EAL4', 'EAL4+', 'EAL5', 'EAL5+', 'EAL6+', 'EAL7', 'EAL7+']\n",
-    "df['highest_security_level'] = df.security_level.map(lambda all_levels: [eal for eal in all_levels if eal.startswith('EAL')] if all_levels else np.nan)\n",
-    "df.highest_security_level = df.highest_security_level.map(lambda x: x[0] if x and isinstance(x, list) else np.nan)\n",
-    "df.highest_security_level = pd.Categorical(df.highest_security_level, categories=eals, ordered=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "faa13c35",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "levels_df = df[(df[\"highest_security_level\"].notna()) & (df[\"directly_affecting\"].notna()) & (df[\"cert_id\"].notna())].copy()\n",
-    "levels_df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2238bb57",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from typing import Dict\n",
-    "\n",
-    "def get_cert_id_security_level(cert_id: str) -> str:\n",
-    "    cert_id_df = df[df[\"cert_id\"] == cert_id]\n",
-    "    \n",
-    "    if cert_id_df.empty:  # we do not have record in main dset for this cert_id\n",
-    "        return None\n",
-    "    \n",
-    "    return cert_id_df.iloc[0][\"highest_security_level\"]\n",
-    "\n",
-    "\n",
-    "def get_levels_of_affected_certs(affected_certs: Set[str]) -> Dict[str, int]:\n",
-    "    result = {}\n",
-    "    \n",
-    "    for affected_cert_id in affected_certs:\n",
-    "        security_level = get_cert_id_security_level(affected_cert_id)\n",
-    "        \n",
-    "        if security_level is None:  # cert_id does not follow condition for levels_df\n",
-    "            continue\n",
-    "            \n",
-    "        result[security_level] = result.get(security_level, 0) + 1\n",
-    "        \n",
-    "    return result\n",
-    "        \n",
-    "\n",
-    "levels_df[\"affecting_security_levels\"] = levels_df[\"directly_affecting\"].apply(get_levels_of_affected_certs)\n",
-    "levels_df.head(20)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "79b1d0da",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "result = {}\n",
-    "\n",
-    "for security_level in eals:\n",
-    "    security_level_list = []\n",
-    "    counter = collections.Counter()\n",
-    "    security_level_df = levels_df[levels_df[\"highest_security_level\"] == security_level][\"affecting_security_levels\"]\n",
-    "    \n",
-    "    for security_dict in security_level_df:\n",
-    "        counter.update(security_dict)\n",
-    "                \n",
-    "    print(f\"Certs with security level {security_level} are directly affecting other certificates with levels: {dict(counter)}\")\n",
-    "        \n",
-    "    result[security_level] = counter"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e3701a93",
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "plt.figure(figsize=(10,5))\n",
-    "plt.rcParams.update({'font.size': 20})\n",
-    "heatmap_result = []\n",
-    "\n",
-    "\n",
-    "for security_level, counter in result.items():\n",
-    "    security_level_list = []\n",
-    "    for security_level_key in eals:\n",
-    "        security_level_list.append(counter.get(security_level_key, 0))\n",
-    "    \n",
-    "    heatmap_result.append(security_level_list)\n",
-    "    \n",
-    "sns.set(style=\"whitegrid\")\n",
-    "ax = sns.heatmap(heatmap_result, xticklabels=eals, yticklabels=eals,cmap=\"Greens\").set_title(\"Archived certs vs. active certs\")\n",
-    "plt.ylabel(\"Specific security level\")\n",
-    "plt.xlabel(\"Security levels affected by specific security level\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "6403ed38",
-   "metadata": {},
-   "source": [
-    "### Basic Analysis of most common category"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ab4ed23d",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "cards_df = df[df[\"category\"] == \"ICs, Smart Cards and Smart Card-Related Devices and Systems\"]\n",
-    "print(f\"There are total {cards_df.shape[0]} rows ICs, Smart Cards and Smart Card-Related Devices and Systems category.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "39cfb0e3",
-   "metadata": {},
-   "source": [
-    "#### How many certificates are active/archived"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cc6bb09a",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "total_archived_certs = sum(cards_df[\"status\"] == \"archived\")\n",
-    "total_active_certs = sum(cards_df[\"status\"] == \"active\")\n",
-    "\n",
-    "print(f\"There are total {total_archived_certs} archived records among smart-cards\")\n",
-    "print(f\"There are total {total_active_certs} active records among smart-cards\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "72f3db18",
-   "metadata": {},
-   "source": [
-    "#### Which manufacturer is the most common in this category?"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1982c2a3",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "most_common_smart_card_manufacturer = cards_df[\"manufacturer\"].value_counts().index[0]\n",
-    "print(f\"The most common manufacturer in smart cards category is: {most_common_smart_card_manufacturer}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "22cd65ee",
-   "metadata": {},
-   "source": [
-    "#### Analysis of security levels of smart-cards"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d47c6bd5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# The most common security level among smart-cards\n",
-    "most_common_sec_level = cards_df[\"highest_security_level\"].value_counts().index[0]\n",
-    "sec_level_amount = cards_df[\"highest_security_level\"].value_counts()[0]\n",
-    "\n",
-    "print(f\"Most common security level among smart-cards is {most_common_sec_level} with {sec_level_amount} occurences.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "581fc32b",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "# The lowest common security level achieved in dataset\n",
-    "security_level_occurences = cards_df[\"highest_security_level\"].value_counts()\n",
-    "filtered_sec_levels = [sec_level for sec_level, count in security_level_occurences.items() if count > 0]\n",
-    "level_numbers = {x: y for x, y in zip(eals, range(len(eals)))}\n",
-    "\n",
-    "lowest_smart_card_security_level = None\n",
-    "lowest_security_level_int = None\n",
-    "\n",
-    "for sec_level in filtered_sec_levels:\n",
-    "    if lowest_security_level_int is None:\n",
-    "        lowest_security_level_int = level_numbers[sec_level]\n",
-    "        lowest_smart_card_security_level = sec_level\n",
-    "    \n",
-    "    if level_numbers[sec_level] < lowest_security_level_int:\n",
-    "        lowest_security_level_int = level_numbers[sec_level]\n",
-    "        lowest_smart_card_security_level = sec_level\n",
-    "        \n",
-    "print(f\"Lowest security level among smart cards in dataset: {lowest_smart_card_security_level}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "022402ff",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# The highest common security level in smart-card dataset\n",
-    "highest_smart_card_security_level = None\n",
-    "highest_security_level_int = None\n",
-    "\n",
-    "for sec_level in filtered_sec_levels:\n",
-    "    if highest_security_level_int is None:\n",
-    "        highest_security_level_int = level_numbers[sec_level]\n",
-    "        highest_smart_card_security_level = sec_level\n",
-    "    \n",
-    "    if level_numbers[sec_level] > highest_security_level_int:\n",
-    "        highest_security_level_int = level_numbers[sec_level]\n",
-    "        highest_smart_card_security_level = sec_level\n",
-    "        \n",
-    "print(f\"Highest security level among smart cards in dataset: {highest_smart_card_security_level}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "7d91f895",
-   "metadata": {},
-   "source": [
-    "#### View data with lowest security level (EAL1+)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2d6a8849",
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "eal1_plus_df = cards_df[cards_df[\"highest_security_level\"] == \"EAL1+\"]\n",
-    "eal1_plus_df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "04ac7242",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "eal1_plus_df[\"scheme\"].value_counts()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "1d580c0d",
-   "metadata": {},
-   "source": [
-    "#### View data with highest security level (EAL7)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "5030f8eb",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "eal7_df = cards_df[cards_df[\"highest_security_level\"] == highest_smart_card_security_level]\n",
-    "eal7_df.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9b577aca",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "eal7_df[\"scheme\"].value_counts()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0452927f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "eal7_df[eal7_df[\"status\"] == \"active\"]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "556ea87d",
-   "metadata": {},
-   "source": [
-    "#### BSI certs in smart cards dataset"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9c685c2e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "bsi_smart_cards_df = cards_df[cards_df[\"is_bsi_cert\"]]\n",
-    "\n",
-    "print(f\"There is total of {bsi_smart_cards_df.shape[0]} BSI records among smart cards\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "462627db",
-   "metadata": {},
-   "source": [
-    "#### Most common security levels among BSI smart card records"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "41736158",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "bsi_smart_cards_df[\"highest_security_level\"].value_counts()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "13a58439",
-   "metadata": {},
-   "source": [
-    "#### ANSSI certs in smart cards dataset\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "941699e7",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "anssi_smart_cards_df = cards_df[cards_df[\"is_anssi_cert\"]]\n",
-    "\n",
-    "print(f\"There is total of {anssi_smart_cards_df.shape[0]} records ANSSI among smart cards\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "be2911f7",
-   "metadata": {},
-   "source": [
-    "#### Most common security levels among ANSSI smart card records "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "454287f5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "anssi_smart_cards_df[\"highest_security_level\"].value_counts()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "7e955c25",
-   "metadata": {},
-   "source": [
-    "#### Smarts cards which expires next year\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "59fbcf9b",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "next_year = datetime.datetime.now().year + 1\n",
-    "\n",
-    "def is_expiring_next_year(series_datetime):\n",
-    "    return series_datetime.year == next_year\n",
-    "\n",
-    "\n",
-    "cards_next_year_expires_df = cards_df[cards_df[\"not_valid_after\"].apply(is_expiring_next_year)]\n",
-    "cards_next_year_expires_df.head()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "aab6cbec",
-   "metadata": {},
-   "source": [
-    "### Which schemes are directly affecting certs with other schemes"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "dc154008",
-   "metadata": {
-    "scrolled": false
-   },
-   "outputs": [],
-   "source": [
-    "df[\"scheme\"].value_counts()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b5758f83",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "scheme_df = df[df[\"directly_affecting\"].notna()]\n",
-    "print(f\"Total of {scheme_df.shape[0]} certs are directly affecting other certs.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ec96a506",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_scheme_from_cert_id(cert_id: str) -> str:\n",
-    "    scheme_list = df[df[\"cert_id\"] == cert_id][\"scheme\"].tolist()\n",
-    "    \n",
-    "    if not scheme_list:\n",
-    "        return None \n",
-    "    \n",
-    "    \n",
-    "    return df[df[\"cert_id\"] == cert_id][\"scheme\"].tolist()[0]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7c537844",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "CC_SCHEMES = [\"US\", \"FR\", \"DE\", \"JP\", \"CA\", \"NL\", \"ES\", \"KR\", \"UK\", \"AU\", \"NO\", \"SE\", \"MY\", \"TR\", \"IT\", \"IN\", \"SG\"]\n",
-    "result = {}\n",
-    "\n",
-    "\n",
-    "for scheme in CC_SCHEMES:\n",
-    "    counter = collections.Counter()\n",
-    "    scheme_affecting_series = scheme_df[scheme_df[\"scheme\"] == scheme][\"directly_affecting\"]\n",
-    "    \n",
-    "    for affecting_set in scheme_affecting_series:\n",
-    "        tmp_dict = {}\n",
-    "        \n",
-    "        for cert_id in affecting_set:\n",
-    "            current_scheme = get_scheme_from_cert_id(cert_id)\n",
-    "            tmp_dict[current_scheme] = tmp_dict.get(current_scheme, 0) + 1\n",
-    "            \n",
-    "        counter.update(tmp_dict)\n",
-    "    \n",
-    "    result[scheme] = counter    \n",
-    "\n",
-    "print(result)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "67c2b3c7",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "plt.figure(figsize=(10,5))\n",
-    "plt.rcParams.update({'font.size': 20})\n",
-    "heatmap_result = []\n",
-    "\n",
-    "\n",
-    "for scheme, counter in result.items():\n",
-    "    print(scheme, counter)\n",
-    "    scheme_list = []\n",
-    "    for scheme_key in CC_SCHEMES:\n",
-    "        scheme_list.append(counter.get(scheme_key, 0))\n",
-    "    \n",
-    "    heatmap_result.append(scheme_list)\n",
-    "    \n",
-    "print(heatmap_result)\n",
-    "sns.set(style=\"whitegrid\")\n",
-    "ax = sns.heatmap(heatmap_result, xticklabels=CC_SCHEMES, yticklabels=CC_SCHEMES,cmap=\"Greens\").set_title(\"Archived certs vs. active certs\")\n",
-    "plt.ylabel(\"Specific cert scheme\")\n",
-    "plt.xlabel(\"Schemes affected by specific scheme\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "cd388919",
-   "metadata": {},
-   "source": [
-    "#### Dependencies among scheme\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c956178b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def return_unique_years_in_dataset():    \n",
-    "    unique_years = set()\n",
-    "\n",
-    "    for timestamp_record in scheme_df[\"not_valid_before\"]:\n",
-    "        unique_years.add(timestamp_record.year)\n",
-    "        \n",
-    "    return unique_years"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "be0b4b80",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "CC_SCHEMES = [\"US\", \"FR\", \"DE\", \"JP\", \"CA\", \"NL\", \"ES\", \"KR\", \"UK\", \"AU\", \"NO\", \"SE\", \"MY\", \"TR\", \"IT\", \"IN\", \"SG\"]\n",
-    "\n",
-    "def discover_scheme_dependiencies_in_dataset(dataset):\n",
-    "    result = {}\n",
-    "\n",
-    "    for scheme in CC_SCHEMES:\n",
-    "        counter = collections.Counter()\n",
-    "        scheme_affecting_series = dataset[dataset[\"scheme\"] == scheme][\"directly_affecting\"]\n",
-    "\n",
-    "        for affecting_set in scheme_affecting_series:\n",
-    "            tmp_dict = {}\n",
-    "\n",
-    "            for cert_id in affecting_set:\n",
-    "                current_scheme = get_scheme_from_cert_id(cert_id)\n",
-    "                tmp_dict[current_scheme] = tmp_dict.get(current_scheme, 0) + 1\n",
-    "\n",
-    "            counter.update(tmp_dict)\n",
-    "\n",
-    "        result[scheme] = counter\n",
-    "\n",
-    "    return result\n",
-    "\n",
-    "discover_scheme_dependiencies_in_dataset(scheme_df)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cc3b31bd",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "CC_SCHEMES = [\"US\", \"FR\", \"DE\", \"JP\", \"CA\", \"NL\", \"ES\", \"KR\", \"UK\", \"AU\", \"NO\", \"SE\", \"MY\", \"TR\", \"IT\", \"IN\", \"SG\"]\n",
-    "\n",
-    "def discover_scheme_dependiencies_in_dataset(year: int):\n",
-    "    result = {}\n",
-    "\n",
-    "    for scheme in CC_SCHEMES:\n",
-    "        counter = collections.Counter()\n",
-    "        current_scheme_df = scheme_df[scheme_df[\"scheme\"] == scheme]  # [\"directly_affecting\"]\n",
-    "        \n",
-    "        for index, row in current_scheme_df.iterrows():\n",
-    "            if row[\"not_valid_before\"].year != year:\n",
-    "                continue\n",
-    "            \n",
-    "            tmp_dict = {}\n",
-    "            for cert_id in row[\"directly_affecting\"]:\n",
-    "                current_scheme = get_scheme_from_cert_id(cert_id)\n",
-    "                tmp_dict[current_scheme] = tmp_dict.get(current_scheme, 0) + 1\n",
-    "\n",
-    "            counter.update(tmp_dict)\n",
-    "\n",
-    "        result[scheme] = counter\n",
-    "\n",
-    "    return result"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "965e6f3e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "UNIQUE_YEARS = return_unique_years_in_dataset()\n",
-    "year_result = {}\n",
-    "\n",
-    "for year in UNIQUE_YEARS:\n",
-    "    scheme_year_df = scheme_df[scheme_df[\"not_valid_before\"] == year]\n",
-    "    year_result[year] = discover_scheme_dependiencies_in_dataset(year)\n",
-    "\n",
-    "print(year_result)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.7"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}