Update the final run notebooks and cert_id eval data.

6 files changed, 124 insertions, 72 deletions

diff --git a/data/cert_id_eval/duplicate_ids.csv b/data/cert_id_eval/duplicate_ids.csv
index 4e08e6a8..b24bc4f3 100644
--- a/data/cert_id_eval/duplicate_ids.csv
+++ b/data/cert_id_eval/duplicate_ids.csv
@@ -137,4 +137,10 @@ ANSSI-CC-2018/58,d57c84efd02c4bb7,tp,ANSSI-CC-2018/58,,
 CCEVS-VR-03-0044,814a2ac0aeae08b1,tp,CCEVS-VR-03-0044,clone,
 CCEVS-VR-03-0044,02a31d0ed42e4ff1,tp,CCEVS-VR-03-0044,clone,
 ANSSI-CC-2018/56,6df8f86449e56c05,tp,ANSSI-CC-2018/56,,
-ANSSI-CC-2018/56,c5e25f90c7006546,fp,ANSSI-CC-2018/56v2,number of mentions of component certificate higher than actual,frontpage match
\ No newline at end of file
+ANSSI-CC-2018/56,c5e25f90c7006546,fp,ANSSI-CC-2018/56v2,number of mentions of component certificate higher than actual,frontpage match
+ANSSI-CC-2020/42,dcb259767b98f381,tp,ANSSI-CC-2020/42,,
+ANSSI-CC-2020/42,4c017087976d05eb,fp,ANSSI-CC-2020/49,number of mentions of component certificate higher than actual,frontpage match
+ANSSI-CC-2021/34,60f31793e1e5f4e1,tp,ANSSI-CC-2021/34,,
+ANSSI-CC-2021/34,3cbdd57de3f32ad0,fp,ANSSI-CC-2021/34-R01,number of mentions of component certificate higher than actual,frontpage match
+ANSSI-CC-2021/29,6ed3cd21c6a2c9d0,tp,ANSSI-CC-2021/29,,
+ANSSI-CC-2021/29,e34879f5664b1908,fp,ANSSI-CC-2021/36,number of mentions of component certificate higher than actual,frontpage match
\ No newline at end of file
diff --git a/data/cert_id_eval/missing_ids.csv b/data/cert_id_eval/missing_ids.csv
index d9c38f5e..163bc6dc 100644
--- a/data/cert_id_eval/missing_ids.csv
+++ b/data/cert_id_eval/missing_ids.csv
@@ -30,7 +30,6 @@ b43100f226a61f53,FR,,malformed PDF
 0a117ebd82bdfb0c,US,,cert_id missing
 95948dc5f3e472fb,IT,,cert_id missing
 9253ec0d31264194,US,,cert_id missing
-1a8ac8fbe5bfcda9,NO,SERTIT-017,404 for report PDF
 3faae83d248da8ba,IT,,cert_id missing
 ae1b80e9cccec491,JP,C0089,pdf issue
 5945cedd8cb01e06,JP,C0120,report and target documents switched
@@ -48,9 +47,7 @@ d79ecebfe21f2144,US,,404 for report PDF
 5e646214ea5b1044,US,CCEVS-VR-VID11243-2022,has security target uploaded in place of report
 d399d47f958a80dd,AU,Certificate Number: 2020/126,certificate.pdf + OCR
 8b357b0ccdafe13b,AU,Certificate Number: 2019/120,certificate.pdf + OCR
-4fb34c6529549e26,JP,C0064,pdf issue
 c64192c1597a1a14,AU,,cert_id missing
-245f78711cfcb537,JP,C0142,pdf issue
 acd95d9dd4ad410c,AU,Certificate Number: 2019/121,certificate.pdf + OCR
 9903c89cbb35eec1,AU,Certificate Number: 2019/123,certificate.pdf + OCR
 908128ea95f94b09,AU,Certificate Number: 2020/132,certificate.pdf + OCR
@@ -59,9 +56,7 @@ acd95d9dd4ad410c,AU,Certificate Number: 2019/121,certificate.pdf + OCR
 5db22161b84ab3d5,AU,Certificate Number: 2020/127,certificate.pdf + OCR
 5390c9458174b933,AU,Certificate Number: 2020/129,certificate.pdf + OCR
 bd7d1fa4b1e938e4,AU,Certificate Number: 2020/128,certificate.pdf + OCR
-bd992ce6d76206bf,JP,C0203,pdf issue
 c74dd9d805e44354,FR,ANSSI-CC-2005/12,pdf issue
-ea708fac8f672c0f,JP,C0702,pdf issue
 14e28608791e4a82,AU,,cert_id missing
 96cf0d5d7f98fbf1,AU,Certificate Number: 2020/130,certificate.pdf + OCR
 770f2b13b0723e49,AU,Certificate Number: 2021/134,certificate.pdf + OCR
@@ -69,12 +64,14 @@ ea708fac8f672c0f,JP,C0702,pdf issue
 7a799fc315ee99da,AU,Certificate Number: 2021/133,certificate.pdf + OCR
 5f6946c674c32b30,AU,Certificate Number: 2022/138,certificate.pdf
 749fae44017016c5,FR,ANSSI-CC-2005/01,pdf issue
-3097b47b6f065ba5,JP,C0062,pdf issue
 9a1c767d358eee50,AU,,cert_id missing
 5ceb859ce72daecf,AU,Certificate Number: 2021/135,certificate.pdf + OCR
 fdf20384e73efb57,AU,Certificate Number: 2020/131,certificate.pdf + OCR
 d2b49eb29a3dcbdc,US,,cert_id missing
 afcf5263d97159f8,AU,Certificate Number: 2019/125,certificate.pdf + OCR
-10d135ab24eab5a8,CA,383-4-270,pdf issue
 4408217c4ce524fd,FR,ANSSI-CC-2003/12,pdf issue
-c3f8e8ddb1dabb02,FR,ANSSI-CC-2003/12,pdf issue
\ No newline at end of file
+c3f8e8ddb1dabb02,FR,ANSSI-CC-2003/12,pdf issue
+739d07a9745954b3,TR,14.10.01/11-211,regex issue
+57039593f41cb794,SE,CSEC201003,regex issue
+cfc4c5b3fbcb0b6f,TR,14.10.01/11-520,regex issue
+f8758e4c8ddc1de5,FR,ANSSI-CC-2004/19,regex issue
\ No newline at end of file
diff --git a/notebooks/cc/references.ipynb b/notebooks/cc/references.ipynb
index eadddb86..489d59f7 100644
--- a/notebooks/cc/references.ipynb
+++ b/notebooks/cc/references.ipynb
@@ -19,7 +19,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -73,7 +73,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -82,12 +82,12 @@
    "outputs": [],
    "source": [
     "# Initialize\n",
-    "dset = CCDataset.from_json(\"../../cc_dset/CommonCriteria_dataset.json\")"
+    "dset = CCDataset.from_json(\"../cc_09_10_2022/cc_new.json\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -118,14 +118,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
     },
     "scrolled": true
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\\newcommand{\\numCcAllDirectReferencing}{1497}\n",
+      "\\newcommand{\\numCcAllNotDirectReferencing}{3632}\n",
+      "\\newcommand{\\numCcWithIdDirectReferencing}{1497}\n",
+      "\\newcommand{\\numCcWithIdNotDirectReferencing}{3556}\n"
+     ]
+    }
+   ],
    "source": [
     "df[\"has_outgoing_direct_references\"] = df.directly_referencing.notnull()\n",
     "df[\"has_incoming_direct_references\"] = df.directly_referenced_by.notnull()\n",
@@ -150,9 +161,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\\newcommand{\\numCCActiveDirectReferencing}{545}\n",
+      "\\newcommand{\\numCCActiveDirectReferencingArchived}{165}\n"
+     ]
+    }
+   ],
    "source": [
     "print(f\"\\\\newcommand{{\\\\numCCActiveDirectReferencing}}{{{df_id_rich.loc[df_id_rich.status == 'active'].has_outgoing_direct_references.sum()}}}\")\n",
     "\n",
@@ -174,7 +194,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -201,13 +221,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\\newcommand{\\numCCDirectRefsSameCategory}{2123}\n",
+      "\\newcommand{\\numCCDirectRefsOtherCategory}{192}\n",
+      "\\newcommand{\\numCCDirectRefs}{2315}\n",
+      "\\newcommand{\\numCCDirectRefsFromSmartcards}{1886}\n"
+     ]
+    }
+   ],
    "source": [
     "cert_id_to_category_mapping = dict(zip(df.cert_id, df.category))\n",
     "cert_id_to_category_mapping[np.NaN] = \"No references\"\n",
@@ -247,7 +278,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -281,13 +312,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of certificates that reference some archived certificate: 929\n"
+     ]
+    }
+   ],
    "source": [
     "def references_archived_cert(references):\n",
     "    if pd.isnull(references):\n",
@@ -326,7 +365,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -359,9 +398,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\\newcommand{\\numCCUSReferencing}{4}\n",
+      "\\newcommand{\\numCCUS}{959}\n"
+     ]
+    }
+   ],
    "source": [
     "print(f\"\\\\newcommand{{\\\\numCCUSReferencing}}{{{len(df_id_rich.loc[(df_id_rich.scheme == 'US') & (df_id_rich.directly_referencing.notnull())])}}}\")\n",
     "print(f\"\\\\newcommand{{\\\\numCCUS}}{{{len(df_id_rich.loc[(df_id_rich.scheme == 'US')])}}}\")"
@@ -378,7 +426,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -403,7 +451,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -423,13 +471,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Certificates in dataset: 5129\n",
+      "Certificates with extracted IDs: 4979\n"
+     ]
+    }
+   ],
    "source": [
     "certs_with_ids = {cert.heuristics.cert_id: cert for cert in dset if cert.heuristics.cert_id}\n",
     "\n",
diff --git a/notebooks/cert_id_eval.ipynb b/notebooks/cert_id_eval.ipynb
index dcd7ae67..65d14225 100644
--- a/notebooks/cert_id_eval.ipynb
+++ b/notebooks/cert_id_eval.ipynb
@@ -58,17 +58,11 @@
     "pycharm": {
      "name": "#%%\n"
     },
-    "scrolled": true
+    "scrolled": false
    },
    "outputs": [],
    "source": [
-    "# dset = CCDataset.from_web_latest()\n",
-    "dset = CCDataset.from_json(\"../cc_dset/CommonCriteria_dataset.json\")\n",
-    "#dset._download_reports(False)\n",
-    "#dset._convert_reports_to_txt(False)\n",
-    "#dset._extract_report_keywords()\n",
-    "#dset._compute_normalized_cert_ids()\n",
-    "#dset.to_json()"
+    "dset = CCDataset.from_json(\"cc_09_10_2022/cc.json\")"
    ]
   },
   {
@@ -131,7 +125,9 @@
    "outputs": [],
    "source": [
     "missing_manual = pd.read_csv(\"../data/cert_id_eval/missing_ids.csv\")\n",
-    "set(missing_manual.id) == missing_id_dgsts"
+    "print(set(missing_manual.id) == missing_id_dgsts)\n",
+    "print(set(missing_manual.id).difference(missing_id_dgsts))\n",
+    "print(set(missing_id_dgsts).difference(missing_manual.id))"
    ]
   },
   {
@@ -304,7 +300,9 @@
    "outputs": [],
    "source": [
     "duplicate_manual = pd.read_csv(\"../data/cert_id_eval/duplicate_ids.csv\")\n",
-    "set(duplicate_manual.id) == duplicate_id_dgsts"
+    "set(duplicate_manual.id) == duplicate_id_dgsts\n",
+    "print(set(duplicate_manual.id).difference(duplicate_id_dgsts))\n",
+    "print(set(duplicate_id_dgsts).difference(duplicate_manual.id))"
    ]
   },
   {
diff --git a/notebooks/final_run.ipynb b/notebooks/final_run.ipynb
index 281691c4..5714ad43 100644
--- a/notebooks/final_run.ipynb
+++ b/notebooks/final_run.ipynb
@@ -9,7 +9,21 @@
     "from sec_certs.dataset import CCDataset\n",
     "from pathlib import Path\n",
     "import shutil\n",
-    "import subprocess"
+    "import subprocess\n",
+    "from sec_certs.utils import helpers\n",
+    "from urllib.parse import unquote_plus, urlparse"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def hash_file(path):\n",
+    "    return helpers.get_sha256_filepath(path)\n",
+    "def extract_filename(link):\n",
+    "    return unquote_plus(str(urlparse(link).path).split(\"/\")[-1])"
    ]
   },
   {
@@ -26,7 +40,7 @@
      "output_type": "stream",
      "text": [
       "True\n",
-      "False\n"
+      "True\n"
      ]
     }
    ],
@@ -200,10 +214,15 @@
     "    if report_pdf.exists():\n",
     "        shutil.copy(report_pdf, cert.state.report_pdf_path)\n",
     "        cert.state.report_download_ok = True\n",
+    "        cert.state.report_pdf_hash = hash_file(cert.state.report_pdf_path)\n",
+    "    cert.pdf_data.report_filename = extract_filename(cert.report_link)\n",
+    "\n",
     "    target_pdf = doc_dump_dir / \"target\" / \"pdf\" / f\"{cert.dgst}.pdf\"\n",
     "    if target_pdf.exists():\n",
     "        shutil.copy(target_pdf, cert.state.st_pdf_path)\n",
-    "        cert.state.st_download_ok = True"
+    "        cert.state.st_download_ok = True\n",
+    "        cert.state.st_pdf_hash = hash_file(cert.state.st_pdf_path)\n",
+    "    cert.pdf_data.st_filename = extract_filename(cert.st_link)"
    ]
   },
   {
@@ -409,13 +428,7 @@
       "Converting targets to txt:  40%|███████████████████████████████████▉                                                      | 2015/5038 [09:13<13:29,  3.73it/s]Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/b4bc79d8a558c7cf.pdf\n",
       "Converting targets to txt:  40%|████████████████████████████████████                                                      | 2020/5038 [09:14<13:55,  3.61it/s]Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/09b43147f1307352.pdf\n",
       "Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/c16a92b40550193d.pdf\n",
-      "Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/ce987167d42db722.pdf\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
+      "Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/ce987167d42db722.pdf\n",
       "Converting targets to txt:  40%|███████████████████████████████████▌                                                    | 2034/5038 [10:29<2:33:49,  3.07s/it]Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/2ff761edd4ed9b72.pdf\n",
       "Converting targets to txt:  41%|███████████████████████████████████▉                                                    | 2054/5038 [12:42<3:10:14,  3.83s/it]Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/9e4d3347efd95ec9.pdf\n",
       "Converting targets to txt:  43%|███████████████████████████████████████                                                   | 2184/5038 [13:32<15:16,  3.11it/s]Detected garbage during conversion of cc_09_10_2022/certs/targets/pdf/ad9ea5a4acf9ac53.pdf\n",
@@ -585,13 +598,7 @@
       "Failed to read metadata of certs/reports/pdf/989b106f5e63b4d1.pdf, error: invalid literal for int() with base 10: b'n'\n",
       "Failed to read metadata of certs/reports/pdf/206662b82595265b.pdf, error: 'NumberObject' object is not subscriptable\n",
       "Failed to read metadata of certs/reports/pdf/0160f6becf137107.pdf, error: 'NumberObject' object is not subscriptable\n",
-      "Failed to read metadata of certs/reports/pdf/9558a9eaadae855e.pdf, error: PDF metadata unavailable\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
+      "Failed to read metadata of certs/reports/pdf/9558a9eaadae855e.pdf, error: PDF metadata unavailable\n",
       "Extracting report metadata:  73%|████████████████████████████████████████████████████████████████▌                       | 3746/5107 [00:30<00:12, 106.48it/s]Failed to read metadata of certs/reports/pdf/9114f0f9938e558c.pdf, error: PDF metadata unavailable\n",
       "Failed to read metadata of certs/reports/pdf/08b09322ee9df0af.pdf, error: PDF metadata unavailable\n",
       "Failed to read metadata of certs/reports/pdf/518b3066ebfabcd3.pdf, error: PDF metadata unavailable\n",
@@ -708,13 +715,7 @@
       "WARNING: multiple rules are matching same certification document: cc_09_10_2022/certs/reports/txt/5c3806bf79eeab7f.txt\n",
       "Extracting report frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████| 5107/5107 [00:14<00:00, 350.74it/s]\n",
       "Extracting target frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████| 5033/5033 [00:19<00:00, 255.18it/s]\n",
-      "Extracting report keywords: 100%|███████████████████████████████████████████████████████████████████████████████████████| 5107/5107 [8:44:06<00:00,  6.16s/it]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
+      "Extracting report keywords: 100%|███████████████████████████████████████████████████████████████████████████████████████| 5107/5107 [8:44:06<00:00,  6.16s/it]\n",
       "Extracting target keywords: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 5033/5033 [34:44<00:00,  2.42it/s]\n",
       "Fitting the CPE classifier: 100%|█████████████████████████████████████████████████████████████████████████████████| 859965/859965 [00:01<00:00, 836963.35it/s]\n",
       "Predicting CPE matches with the classifier:  33%|███████████████████████▉                                                 | 1686/5129 [00:53<01:43, 33.41it/s]"
@@ -822,13 +823,6 @@
     "# Finally, dump it all\n",
     "dset.to_json()"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
diff --git a/sec_certs/rules.yaml b/sec_certs/rules.yaml
index 8a4cd022..87589d72 100644
--- a/sec_certs/rules.yaml
+++ b/sec_certs/rules.yaml
@@ -61,7 +61,7 @@ cc_cert_id:
     - "[0-9\\.]+?/TSE-CCCS-[0-9]+"  # Turkish CCCS (21.0.0sc/TSE-CCCS-75)
     - "(?:[0-9]{1,2}\\.){2}[0-9]{1,2}/[0-9]{1,4}-[0-9]{3}"   # 21.0.01/13-028
   SE:
-    - "CSEC ?[0-9]{7}"  # Sweden (CSEC2019015)
+    - "CSEC ?[0-9]{6,7}"  # Sweden (CSEC2019015)
   IN:
     # India (IC3S/DEL01/VALIANT/EAL1/0317/0007/CR  STQC/CC/14-15/12/ETR/0017 IC3S/MUM01/CISCO/cPP/0119/0016/CR)
     # will miss STQC/CC/14-15/12/ETR/0017