refactor: General refactoring of level references

1 files changed, 112 insertions, 132 deletions

diff --git a/notebooks/fips/references.ipynb b/notebooks/fips/references.ipynb
index b646a874..d54fb56f 100644
--- a/notebooks/fips/references.ipynb
+++ b/notebooks/fips/references.ipynb
@@ -12,7 +12,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 62,
+   "execution_count": 1,
    "id": "91a0cb1b",
    "metadata": {},
    "outputs": [],
@@ -71,7 +71,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Downloading FIPS Dataset: 100%|█████████████████████████████████████| 57.1M/57.1M [00:06<00:00, 9.02MB/s]\n"
+      "Downloading FIPS Dataset: 100%|█████████████████████████████████████| 57.1M/57.1M [00:06<00:00, 8.73MB/s]\n"
      ]
     }
    ],
@@ -172,7 +172,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 59,
    "id": "eca9eb4d",
    "metadata": {},
    "outputs": [
@@ -216,7 +216,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 60,
    "id": "ac66ad0f",
    "metadata": {},
    "outputs": [
@@ -424,7 +424,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 54,
+   "execution_count": 14,
    "id": "678835dd",
    "metadata": {
     "scrolled": false
@@ -455,7 +455,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 58,
+   "execution_count": 15,
    "id": "3c17e838",
    "metadata": {
     "scrolled": false
@@ -477,7 +477,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 61,
+   "execution_count": 16,
    "id": "6ca75914",
    "metadata": {},
    "outputs": [
@@ -561,15 +561,15 @@
        "      <td>Multi-Chip Stand Alone</td>\n",
        "      <td>2020-02-19</td>\n",
        "      <td>NaT</td>\n",
-       "      <td>{ECDSA#C786, DRBG#C786, RSA#C786, KTS#C786, HM...</td>\n",
+       "      <td>{KTS#C794, SHS#C786, DSA#C786, RSA#C786, ECDSA...</td>\n",
        "      <td>...</td>\n",
-       "      <td>{CVE-2016-0647, CVE-2016-1908, CVE-2016-2793, ...</td>\n",
-       "      <td>{3699, 4135, 3893, 4146, 3921, 4147, 4164}</td>\n",
-       "      <td>{3699, 4135, 3893, 4146, 3921, 4147, 4164}</td>\n",
+       "      <td>{CVE-2016-6302, CVE-2016-3521, CVE-2016-4448, ...</td>\n",
+       "      <td>{4146, 4164, 4147, 3699, 3893, 4135, 3921}</td>\n",
+       "      <td>{4146, 4164, 4147, 3699, 3893, 4135, 3921}</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
-       "      <td>{3893, 3699, 3921}</td>\n",
-       "      <td>{3893, 3699, 3921}</td>\n",
+       "      <td>{3893, 3921, 3699}</td>\n",
+       "      <td>{3893, 3921, 3699}</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "      <td>2020</td>\n",
@@ -585,17 +585,17 @@
        "      <td>Multi-Chip Stand Alone</td>\n",
        "      <td>2010-08-12</td>\n",
        "      <td>NaT</td>\n",
-       "      <td>{RNG#649, Triple-DES#846, AES GMAC#1168, ECDSA...</td>\n",
+       "      <td>{HMAC#686, Triple-DES#846, DSA#391, AES#1168, ...</td>\n",
        "      <td>...</td>\n",
-       "      <td>{CVE-2009-0081, CVE-2009-0320, CVE-2009-1536, ...</td>\n",
-       "      <td>{1805, 1808, 1806, 1817, 1807, 1339, 2736}</td>\n",
-       "      <td>{1805, 1808, 1806, 1817, 1807, 1339, 2736}</td>\n",
+       "      <td>{CVE-2009-0320, CVE-2008-2250, CVE-2010-0025, ...</td>\n",
+       "      <td>{1806, 1805, 2736, 1808, 1817, 1807, 1339}</td>\n",
+       "      <td>{1806, 1805, 2736, 1808, 1817, 1807, 1339}</td>\n",
        "      <td>{1334}</td>\n",
-       "      <td>{1333, 1321, 1334}</td>\n",
-       "      <td>{1805, 1808, 1806, 1817, 1807, 1339}</td>\n",
-       "      <td>{1805, 1808, 1806, 1817, 1807, 1339}</td>\n",
+       "      <td>{1334, 1321, 1333}</td>\n",
+       "      <td>{1806, 1805, 1808, 1817, 1807, 1339}</td>\n",
+       "      <td>{1806, 1805, 1808, 1817, 1807, 1339}</td>\n",
        "      <td>{1334}</td>\n",
-       "      <td>{1333, 1321, 1334}</td>\n",
+       "      <td>{1334, 1321, 1333}</td>\n",
        "      <td>2010</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
@@ -621,43 +621,43 @@
        "\n",
        "                                                         algorithms  ...  \\\n",
        "dgst                                                                 ...   \n",
-       "10acb87b3040f4e1  {ECDSA#C786, DRBG#C786, RSA#C786, KTS#C786, HM...  ...   \n",
-       "3c4ee858b268931a  {RNG#649, Triple-DES#846, AES GMAC#1168, ECDSA...  ...   \n",
+       "10acb87b3040f4e1  {KTS#C794, SHS#C786, DSA#C786, RSA#C786, ECDSA...  ...   \n",
+       "3c4ee858b268931a  {HMAC#686, Triple-DES#846, DSA#391, AES#1168, ...  ...   \n",
        "\n",
        "                                                       related_cves  \\\n",
        "dgst                                                                  \n",
-       "10acb87b3040f4e1  {CVE-2016-0647, CVE-2016-1908, CVE-2016-2793, ...   \n",
-       "3c4ee858b268931a  {CVE-2009-0081, CVE-2009-0320, CVE-2009-1536, ...   \n",
+       "10acb87b3040f4e1  {CVE-2016-6302, CVE-2016-3521, CVE-2016-4448, ...   \n",
+       "3c4ee858b268931a  {CVE-2009-0320, CVE-2008-2250, CVE-2010-0025, ...   \n",
        "\n",
        "                               module_directly_referenced_by  \\\n",
        "dgst                                                           \n",
-       "10acb87b3040f4e1  {3699, 4135, 3893, 4146, 3921, 4147, 4164}   \n",
-       "3c4ee858b268931a  {1805, 1808, 1806, 1817, 1807, 1339, 2736}   \n",
+       "10acb87b3040f4e1  {4146, 4164, 4147, 3699, 3893, 4135, 3921}   \n",
+       "3c4ee858b268931a  {1806, 1805, 2736, 1808, 1817, 1807, 1339}   \n",
        "\n",
        "                             module_indirectly_referenced_by  \\\n",
        "dgst                                                           \n",
-       "10acb87b3040f4e1  {3699, 4135, 3893, 4146, 3921, 4147, 4164}   \n",
-       "3c4ee858b268931a  {1805, 1808, 1806, 1817, 1807, 1339, 2736}   \n",
+       "10acb87b3040f4e1  {4146, 4164, 4147, 3699, 3893, 4135, 3921}   \n",
+       "3c4ee858b268931a  {1806, 1805, 2736, 1808, 1817, 1807, 1339}   \n",
        "\n",
        "                 module_directly_referencing module_indirectly_referencing  \\\n",
        "dgst                                                                         \n",
        "10acb87b3040f4e1                         NaN                           NaN   \n",
-       "3c4ee858b268931a                      {1334}            {1333, 1321, 1334}   \n",
+       "3c4ee858b268931a                      {1334}            {1334, 1321, 1333}   \n",
        "\n",
        "                         policy_directly_referenced_by  \\\n",
        "dgst                                                     \n",
-       "10acb87b3040f4e1                    {3893, 3699, 3921}   \n",
-       "3c4ee858b268931a  {1805, 1808, 1806, 1817, 1807, 1339}   \n",
+       "10acb87b3040f4e1                    {3893, 3921, 3699}   \n",
+       "3c4ee858b268931a  {1806, 1805, 1808, 1817, 1807, 1339}   \n",
        "\n",
        "                       policy_indirectly_referenced_by  \\\n",
        "dgst                                                     \n",
-       "10acb87b3040f4e1                    {3893, 3699, 3921}   \n",
-       "3c4ee858b268931a  {1805, 1808, 1806, 1817, 1807, 1339}   \n",
+       "10acb87b3040f4e1                    {3893, 3921, 3699}   \n",
+       "3c4ee858b268931a  {1806, 1805, 1808, 1817, 1807, 1339}   \n",
        "\n",
        "                 policy_directly_referencing policy_indirectly_referencing  \\\n",
        "dgst                                                                         \n",
        "10acb87b3040f4e1                         NaN                           NaN   \n",
-       "3c4ee858b268931a                      {1334}            {1333, 1321, 1334}   \n",
+       "3c4ee858b268931a                      {1334}            {1334, 1321, 1333}   \n",
        "\n",
        "                 year_from  \n",
        "dgst                        \n",
@@ -667,7 +667,7 @@
        "[2 rows x 23 columns]"
       ]
      },
-     "execution_count": 61,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -686,7 +686,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 17,
    "id": "b01d5d38",
    "metadata": {},
    "outputs": [],
@@ -705,19 +705,14 @@
     "        print(f\"Error Cert ID: {cert_id} has {sub_df.shape[0]} occurrences.\")\n",
     "        return None\n",
     "    \n",
-    "    return sub_df.iloc[0][column]\n",
-    "\n",
-    "def is_cert_referencing_historical_cert(dataset: DataFrame, cert_ids: set[int]) -> bool:\n",
-    "    return \"historical\" in [get_cert_property(dataset, cert_id, \"status\") for cert_id in cert_ids]"
+    "    return sub_df.iloc[0][column]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
-   "id": "9c05ffb2",
-   "metadata": {
-    "scrolled": true
-   },
+   "execution_count": 58,
+   "id": "670cc557",
+   "metadata": {},
    "outputs": [
     {
      "name": "stdout",
@@ -731,12 +726,15 @@
    ],
    "source": [
     "active_certs_referencing_historical: list[int] = []\n",
+    "# NOTE: \"cert_id\" field is represented as integer, however \"module_directly_referencing\" is set of strings\n",
+    "historical_cert_ids: set[str] = set(df[df[\"status\"] == \"historical\"][\"cert_id\"].apply(str))\n",
     "active_referencing_certs: DataFrame = refs_df[(refs_df[\"status\"] == \"active\") & (refs_df[\"module_directly_referencing\"].notna())]\n",
     "\n",
     "for _, rows in active_referencing_certs.iterrows():\n",
     "    referencing_ids = rows[\"module_directly_referencing\"]\n",
-    "        \n",
-    "    if is_cert_referencing_historical_cert(refs_df, referencing_ids):\n",
+    "    intersection = referencing_ids & historical_cert_ids\n",
+    "    \n",
+    "    if intersection:\n",
     "        active_certs_referencing_historical.append(rows.cert_id)\n",
     "\n",
     "hit_ratio: float = len(active_certs_referencing_historical) / len(active_referencing_certs)\n",
@@ -755,19 +753,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 93,
+   "execution_count": 34,
    "id": "f3402d08",
    "metadata": {
     "scrolled": true
    },
    "outputs": [
     {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Active certificate 3939 is referencing historical certificate 3860 with assigned CVE\n"
+     ]
+    },
+    {
      "data": {
       "text/plain": [
        "[(3939, 3860)]"
       ]
      },
-     "execution_count": 93,
+     "execution_count": 34,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -784,6 +789,7 @@
     "        related_cves = get_cert_property(df, referenced_cert_id_int, \"related_cves\")\n",
     "        \n",
     "        if not pd.isna(related_cves):\n",
+    "            print(f\"Active certificate {cert_id} is referencing historical certificate {referenced_cert_id} with assigned CVE\")\n",
     "            active_cert_referencing_historical_with_cves.append((cert_id, referenced_cert_id_int))\n",
     "            \n",
     "\n",
@@ -791,29 +797,6 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 78,
-   "id": "2a66a006",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "dgst\n",
-       "86d3ce8c4f7dcd36    {CVE-2003-0370, CVE-2004-0902, CVE-2001-0128, ...\n",
-       "Name: related_cves, dtype: object"
-      ]
-     },
-     "execution_count": 78,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df[df[\"cert_id\"] == 3860][\"related_cves\"]"
-   ]
-  },
-  {
    "cell_type": "markdown",
    "id": "d343e55b",
    "metadata": {},
@@ -823,44 +806,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 76,
    "id": "b8225902",
    "metadata": {
     "scrolled": false
    },
    "outputs": [],
    "source": [
-    "referencing_certs: DataFrame = refs_df[refs_df[\"module_directly_referencing\"].notna()]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "7ab480c7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def compare_levels(dataset: DataFrame, cert_ids: set[int], compare_to_level: int) -> int:\n",
-    "    is_referencing_any_lower_level = [get_cert_property(dataset, cert_id, \"level\") < compare_to_level for cert_id in cert_ids]\n",
-    "    return any(is_referencing_any_lower_level)"
+    "referencing_certs: DataFrame = refs_df[refs_df[\"module_directly_referencing\"].notna()]\n",
+    "cert_level_ids: dict[int, set[str]] = {}\n",
+    "unique_levels: list[int] = df[\"level\"].apply(int).unique()\n",
+    "\n",
+    "for level in df[\"level\"].unique():\n",
+    "    level_cert_ids: set[str] = set(df[df[\"level\"] == level][\"cert_id\"].apply(str))\n",
+    "    cert_level_ids[int(level)] = level_cert_ids"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
-   "id": "29c80ecd",
+   "execution_count": 84,
+   "id": "d674de0a",
    "metadata": {},
    "outputs": [],
    "source": [
-    "def get_certs_referencing_lower_level(refs_df: DataFrame, referencing_certs: DataFrame, compare_level: int) -> list[int]:\n",
+    "def get_cert_ids_referencing_lower_level_cert(level_referencing_certs_df: DataFrame, lower_cert_ids: set[str]) -> list[int]:\n",
     "    cert_ids = []\n",
     "    \n",
-    "    for _, rows in referencing_certs.iterrows():\n",
-    "        referencing_ids = rows[\"module_directly_referencing\"]\n",
-    "        if compare_levels(refs_df, referencing_ids, compare_level):\n",
-    "            cert_ids.append(rows.cert_id)\n",
-    "\n",
-    "    return cert_ids"
+    "    for _, cert in level_referencing_certs_df.iterrows():\n",
+    "        if cert[\"module_directly_referencing\"] & lower_cert_ids:\n",
+    "            cert_ids.append(cert[\"cert_id\"])\n",
+    "    \n",
+    "    return cert_ids            "
    ]
   },
   {
@@ -873,11 +849,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
-   "id": "e3a4e349",
-   "metadata": {
-    "scrolled": true
-   },
+   "execution_count": 96,
+   "id": "2b0a8fa1",
+   "metadata": {},
    "outputs": [
     {
      "name": "stdout",
@@ -890,12 +864,14 @@
     }
    ],
    "source": [
-    "compare_level: int = 2\n",
-    "level2_ref_certs = referencing_certs[referencing_certs[\"level\"] == compare_level]\n",
-    "certificates_referencing_lower_level: list[int] = get_certs_referencing_lower_level(refs_df, level2_ref_certs, compare_level)\n",
-    "ratio: float = len(certificates_referencing_lower_level) / level2_ref_certs.shape[0]\n",
-    "print(f\"Total amount of certificates with level {compare_level} referencing at least one certificate: {level2_ref_certs.shape[0]}\")\n",
-    "print(f\"Total amount of certicates with reference at least one certificate with lower level: {len(certificates_referencing_lower_level)}\")\n",
+    "LEVEL2: int = 2 \n",
+    "below_level2_cert_ids: set[str] = cert_level_ids[1]\n",
+    "level2_ref_certs = referencing_certs[referencing_certs[\"level\"] == LEVEL2]\n",
+    "level2_referencing_lower_level = get_cert_ids_referencing_lower_level_cert(level2_ref_certs, below_level2_cert_ids)\n",
+    "ratio: float = len(level2_referencing_lower_level) / level2_ref_certs.shape[0]\n",
+    "\n",
+    "print(f\"Total amount of certificates with level {LEVEL2} referencing at least one certificate: {level2_ref_certs.shape[0]}\")\n",
+    "print(f\"Total amount of certicates with reference at least one certificate with lower level: {len(level2_referencing_lower_level)}\")\n",
     "print(f\"Hit ratio: {round(ratio, 2)}\")"
    ]
   },
@@ -909,8 +885,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
-   "id": "e83b845b",
+   "execution_count": 95,
+   "id": "66576beb",
    "metadata": {},
    "outputs": [
     {
@@ -924,12 +900,14 @@
     }
    ],
    "source": [
-    "compare_level: int = 3\n",
-    "level3_ref_certs = referencing_certs[referencing_certs[\"level\"] == compare_level]\n",
-    "certificates_referencing_lower_level: list[int] = get_certs_referencing_lower_level(refs_df, level3_ref_certs, compare_level)\n",
-    "ratio: float = len(certificates_referencing_lower_level) / level3_ref_certs.shape[0]\n",
-    "print(f\"Total amount of certificates with level {compare_level} referencing at least one certificate: {level3_ref_certs.shape[0]}\")\n",
-    "print(f\"Total amount of certicates with reference at least one certificate with lower level: {len(certificates_referencing_lower_level)}\")\n",
+    "LEVEL3: int = 3\n",
+    "below_level3_cert_ids = below_level2_cert_ids | cert_level_ids[2]\n",
+    "level3_ref_certs = referencing_certs[referencing_certs[\"level\"] == LEVEL3]\n",
+    "level3_referencing_lower_level = get_cert_ids_referencing_lower_level_cert(level3_ref_certs, below_level3_cert_ids)\n",
+    "ratio: float = len(level3_referencing_lower_level) / level3_ref_certs.shape[0]\n",
+    "\n",
+    "print(f\"Total amount of certificates with level {LEVEL3} referencing at least one certificate: {level3_ref_certs.shape[0]}\")\n",
+    "print(f\"Total amount of certicates with reference at least one certificate with lower level: {len(level3_referencing_lower_level)}\")\n",
     "print(f\"Hit ratio: {round(ratio, 2)}\")"
    ]
   },
@@ -943,7 +921,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 97,
    "id": "91db647f",
    "metadata": {
     "scrolled": true
@@ -958,13 +936,13 @@
     }
    ],
    "source": [
-    "compare_level: int = 4\n",
-    "level4_ref_certs = referencing_certs[referencing_certs[\"level\"] == compare_level]\n",
-    "total_level4_certs: int = df[df[\"level\"] == 4].shape[0]\n",
-    "certificates_referencing_lower_level: list[int] = get_certs_referencing_lower_level(refs_df, level4_ref_certs, compare_level)\n",
+    "LEVEL4: int = 4\n",
+    "below_level4_cert_ids = below_level3_cert_ids | cert_level_ids[3]\n",
+    "level4_ref_certs = referencing_certs[referencing_certs[\"level\"] == LEVEL4]\n",
+    "level4_referencing_lower_level = get_cert_ids_referencing_lower_level_cert(level4_ref_certs, below_level4_cert_ids)\n",
     "\n",
     "# TODO - implement the analysis and function needs to cope with empty df or something like that\n",
-    "print(f\"Total amount of certificates with level {compare_level} referencing at least one certificate: {level4_ref_certs.shape[0]}\")\n"
+    "print(f\"Total amount of certificates with level {compare_level} referencing at least one certificate: {level4_ref_certs.shape[0]}\")"
    ]
   },
   {
@@ -977,7 +955,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 27,
    "id": "34b5c83b",
    "metadata": {},
    "outputs": [],
@@ -996,7 +974,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 28,
    "id": "6e69410d",
    "metadata": {
     "scrolled": true
@@ -1027,7 +1005,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 29,
    "id": "0737ca41",
    "metadata": {},
    "outputs": [
@@ -1037,7 +1015,7 @@
        "Text(0.5, 1.0, 'Direct references among FIPS embodiments')"
       ]
      },
-     "execution_count": 24,
+     "execution_count": 29,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1071,7 +1049,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 30,
    "id": "72fb288b",
    "metadata": {},
    "outputs": [],
@@ -1090,7 +1068,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 31,
    "id": "79c3b7c8",
    "metadata": {
     "scrolled": false
@@ -1127,7 +1105,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 32,
    "id": "6694e4cd",
    "metadata": {},
    "outputs": [
@@ -1137,7 +1115,7 @@
        "Text(0.5, 1.0, 'Direct references among FIPS certificate types')"
       ]
      },
-     "execution_count": 27,
+     "execution_count": 32,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -1174,7 +1152,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 33,
    "id": "77a1db87",
    "metadata": {
     "scrolled": false
@@ -1212,9 +1190,11 @@
    ]
   },
   {
-   "cell_type": "markdown",
-   "id": "d3ed0d7c",
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7a8b6cba",
    "metadata": {},
+   "outputs": [],
    "source": []
   }
  ],