Normalize cert_id candidates to 0.0-1.0 range but without sum normalization.

5 files changed, 87 insertions, 35 deletions

diff --git a/data/cert_id_eval/duplicate_ids.csv b/data/cert_id_eval/duplicate_ids.csv
index a8590933..3c7e750a 100644
--- a/data/cert_id_eval/duplicate_ids.csv
+++ b/data/cert_id_eval/duplicate_ids.csv
@@ -222,3 +222,7 @@ BSI-DSZ-CC-0639-2010,2f9feaf4121720da,tp,BSI-DSZ-CC-0639-2010,,
 BSI-DSZ-CC-0639-2010,db7627e56b8621b1,fp,KECS-ISIS-0394-2012,number of mentions of component certificate higher than actual,scheme mismatch
 Certification Report 98/94,28bb2a8869397c4c,fp,CRP106,number of mentions of component certificate higher than actual,scheme mismatch
 Certification Report 98/94,28a9fc2137d72ce9,fp,CRP103,number of mentions of component certificate higher than actual,scheme mismatch
+CSEC2015006,a5607124c92c10ed,tp,CSEC2015006,shared cert_id,
+CSEC2015006,f89c3a438204a45b,tp,CSEC2015006,shared cert_id,
+CRP208,b2aefe8b1bd4d9b3,fp,CRP208A,regex issue,regex fix
+CRP208,c90ddb9915291718,tp,CRP208,,
diff --git a/data/cert_id_eval/missing_ids.csv b/data/cert_id_eval/missing_ids.csv
index 57f00eb4..75da4baf 100644
--- a/data/cert_id_eval/missing_ids.csv
+++ b/data/cert_id_eval/missing_ids.csv
@@ -33,7 +33,7 @@ a91b4c64329d05a8,SE,CSEC2019010,space in cert_id
 c5bad5b309469e22,CA,519-EWA,regex missing
 08e892fc32b0579f,ES,INF-3483,regex missing
 876cb2fe02017f3f,JP,C0012,not parsed due to PDF issue
-0dcc691ea04d1b52,CA,,cert_id missing
+0dcc691ea04d1b52,CA,383-4-351,certificate.docx
 f9c8da9deff77ab5,CA,,report not a PDF (docx) also cert_id missing
 ace7ea6d7f58dbb1,DE,,404 for report PDF
 7147c2f70d983d57,TR,21.0.03/TSE-CCCS-44,report not a PDF (docx)
@@ -79,13 +79,41 @@ d79ecebfe21f2144,US,,404 for report PDF
 7e32023021d5aad2,ES,INF-510,regex missing
 02958808b57334f0,SE,CSEC2019016,report not a PDF (docx)
 391c8b9254ed6ed9,CA,,404 for report PDF
-d3b7b0df0108a466,CA,,cert_id missing
-b17b7e4876e9e482,AU,,
+d3b7b0df0108a466,CA,383-4-344,certificate.docx
 82fcce2db9ef6063,SG,,404 for report PDF
 5e646214ea5b1044,US,CCEVS-VR-VID11243-2022,has security target uploaded in place of report
-2b2268271ee16bd7,FR,ANSSI-CC-2022/05,not parsed due to PDF issue
-e37e1f81d12b63fd,FR,ANSSI-CC-2022/04,not parsed due to PDF issue
-8b6d2aa804720833,FR,ANSSI-CC-2017/42v2,not parsed due to PDF issue
 614f4dcdee0128bd,IN,IC3S/KOL01/ECITelecom/EAL2/0420/0019/CR,space in cert_id
 3477723044183b31,FR,ANSSI-CC-2017/42v2-S01,regex missing
-a5c2f0adcecf1a33,MY,ISCB-5-RPT-C127-CR-v1.1,regex missing
\ No newline at end of file
+a5c2f0adcecf1a33,MY,ISCB-5-RPT-C127-CR-v1.1,regex missing
+d399d47f958a80dd,AU,Certificate Number: 2020/126,certificate.pdf + OCR
+8b357b0ccdafe13b,AU,Certificate Number: 2019/120,certificate.pdf + OCR
+4fb34c6529549e26,JP,C0064,?
+c64192c1597a1a14,AU,,cert_id missing
+245f78711cfcb537,JP,C0142,?
+acd95d9dd4ad410c,AU,Certificate Number: 2019/121,certificate.pdf + OCR
+9903c89cbb35eec1,AU,Certificate Number: 2019/123,certificate.pdf + OCR
+908128ea95f94b09,AU,Certificate Number: 2020/132,certificate.pdf + OCR
+0597940a3fd25600,AU,Certificate Number: 2019/122,certificate.pdf + OCR
+372c24701fa80b5a,AU,Certificate Number: 2019/124,certificate.pdf + OCR
+5db22161b84ab3d5,AU,Certificate Number: 2020/127,certificate.pdf + OCR
+5390c9458174b933,AU,Certificate Number: 2020/129,certificate.pdf + OCR
+bd7d1fa4b1e938e4,AU,Certificate Number: 2020/128,certificate.pdf + OCR
+bd992ce6d76206bf,JP,C0203,?
+c74dd9d805e44354,FR,ANSSI-CC-2005/12,?
+ea708fac8f672c0f,JP,C0702,?
+14e28608791e4a82,AU,,cert_id missing
+96cf0d5d7f98fbf1,AU,Certificate Number: 2020/130,certificate.pdf + OCR
+770f2b13b0723e49,AU,Certificate Number: 2021/134,certificate.pdf + OCR
+3891a6c29ed701b4,FR,ANSSI-CC-2005/02,?
+7a799fc315ee99da,AU,Certificate Number: 2021/133,certificate.pdf + OCR
+5f6946c674c32b30,AU,Certificate Number: 2022/138,certificate.pdf
+749fae44017016c5,FR,ANSSI-CC-2005/01,?
+3097b47b6f065ba5,JP,C0062,?
+9a1c767d358eee50,AU,,cert_id missing
+5ceb859ce72daecf,AU,Certificate Number: 2021/135,certificate.pdf + OCR
+fdf20384e73efb57,AU,Certificate Number: 2020/131,certificate.pdf + OCR
+d2b49eb29a3dcbdc,US,,cert_id missing
+afcf5263d97159f8,AU,Certificate Number: 2019/125,certificate.pdf + OCR
+10d135ab24eab5a8,CA,383-4-270,?
+4408217c4ce524fd,FR,ANSSI-CC-2003/12,?
+c3f8e8ddb1dabb02,FR,ANSSI-CC-2003/12,?
\ No newline at end of file
diff --git a/notebooks/cert_id_eval.ipynb b/notebooks/cert_id_eval.ipynb
index d7b571ce..4bf53f0c 100644
--- a/notebooks/cert_id_eval.ipynb
+++ b/notebooks/cert_id_eval.ipynb
@@ -48,7 +48,7 @@
    "source": [
     "# dset = CCDataset.from_web_latest()\n",
     "dset = CCDataset.from_json(\"../cc_dset/CommonCriteria_dataset.json\")\n",
-    "#dset._extract_pdf_metadata()\n",
+    "#dset._extract_report_metadata()\n",
     "#dset._extract_pdf_frontpage()\n",
     "dset._extract_report_keywords()\n",
     "dset._compute_normalized_cert_ids()\n",
@@ -91,7 +91,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Check manually evaluated missing\n"
+    "### Check manually evaluated missing\n",
+    "\n",
+    "The following cell lists certificates in manual analysis that are still missing an ID."
    ]
   },
   {
@@ -108,11 +110,8 @@
     "with open(\"../data/cert_id_eval/missing_ids.csv\", \"r\") as f:\n",
     "    reader = csv.DictReader(f)\n",
     "    for line in reader:\n",
-    "        try:\n",
-    "            cert = dset[line[\"id\"]]\n",
-    "        except:\n",
-    "            continue\n",
-    "        if line[\"cert_id\"] and line[\"cert_id\"] != cert.heuristics.cert_id:\n",
+    "        cert = dset[line[\"id\"]]\n",
+    "        if line[\"id\"] in missing_id_dgsts:\n",
     "            i += 1\n",
     "            print(line[\"id\"], line[\"cert_id\"], cert.heuristics.cert_id, line[\"reason\"])\n",
     "print(f\"Total: {i}\")"
@@ -298,7 +297,9 @@
    "outputs": [],
    "source": [
     "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n",
-    "print(len(duplicate_ids_issue))"
+    "print(len(duplicate_ids_issue))\n",
+    "for id in duplicate_ids_issue:\n",
+    "    print(id, dset[id].heuristics.cert_id)"
    ]
   },
   {
@@ -356,7 +357,8 @@
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
-    }
+    },
+    "scrolled": false
    },
    "outputs": [],
    "source": [
@@ -414,6 +416,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "correct = set()\n",
@@ -434,25 +441,38 @@
     "        else:\n",
     "            correct.add(line[\"id\"])\n",
     "print(len(correct), len(possible), len(impossible))"
-   ],
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
-    }
-   }
+    },
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "dset[\"0cbcee9f1cde47a8\"].pdf_data.candidate_cert_ids(\"FR\")"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {},
    "outputs": [],
-   "source": [],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   "source": [
+    "from sec_certs.sample.cc_certificate_id import canonicalize\n",
+    "canonicalize('ANSSI-CC-2021/34-R01', \"FR\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -476,4 +496,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 1
-}
\ No newline at end of file
+}
diff --git a/sec_certs/sample/cc_certificate_id.py b/sec_certs/sample/cc_certificate_id.py
index 28a40978..28247826 100644
--- a/sec_certs/sample/cc_certificate_id.py
+++ b/sec_certs/sample/cc_certificate_id.py
@@ -21,7 +21,7 @@ class CertificateId:
         ]
         for rule in rules:
             if match := re.match(rule, new_cert_id):
-                return "ANSSI-CC-" + match.group(1).replace("_", "/").replace("-", "/")
+                return "ANSSI-CC-" + match.group(1).replace("_", "/")
 
         return new_cert_id
 
diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py
index bb7788c9..cb6bcb71 100644
--- a/sec_certs/sample/common_criteria.py
+++ b/sec_certs/sample/common_criteria.py
@@ -308,7 +308,7 @@ class CommonCriteriaCert(
                     matches[cert_id] += 1
             if not matches:
                 return {}
-            total = sum(matches.values())
+            total = max(matches.values())
             results = {}
             for candidate, count in matches.items():
                 results[candidate] = count / total
@@ -330,7 +330,7 @@ class CommonCriteriaCert(
             matches: Counter = Counter(cert_id_matches[scheme])
             if not matches:
                 return {}
-            total = sum(matches.values())
+            total = max(matches.values())
             results = {}
             for candidate, count in matches.items():
                 results[candidate] = count / total
@@ -355,7 +355,7 @@ class CommonCriteriaCert(
                         matches[cert_id] += 1
             if not matches:
                 return {}
-            total = sum(matches.values())
+            total = max(matches.values())
             results = {}
             for candidate, count in matches.items():
                 results[candidate] = count / total
@@ -368,16 +368,16 @@ class CommonCriteriaCert(
             filename_id = self.filename_cert_id(scheme)
             keywords_id = self.keywords_cert_id(scheme)
 
-            # Join them and weigh them, each is normalized to sum weights to 1 (if anything is returned)
+            # Join them and weigh them, each is normalized with weights from 0 to 1 (if anything is returned)
             candidates: Dict[str, float] = defaultdict(lambda: 0.0)
             for candidate, count in frontpage_id.items():
                 candidates[candidate] += count * 1.5
             for candidate, count in metadata_id.items():
                 candidates[candidate] += count * 1.2
-            for candidate, count in filename_id.items():
-                candidates[candidate] += count * 1.2
             for candidate, count in keywords_id.items():
                 candidates[candidate] += count * 1.0
+            for candidate, count in filename_id.items():
+                candidates[candidate] += count * 1.0
             return candidates
 
     @dataclass