Improve CC scheme extraction and matching.

This significantly improves the CC scheme extraction by:

 - Fixing the extraction of several schemes that were mixing
   certified and archived entries by accident.
 - Improving the extraction of cert_ids from scheme sites.
 - Improving the matching heuristic to consider more attributes
   that are usually present in the site data.

Also adds an evaluation notebook to see how this performs.

4 files changed, 20 insertions, 26 deletions

diff --git a/src/sec_certs/utils/extract.py b/src/sec_certs/utils/extract.py
index 340ad568..6af7f4cc 100644
--- a/src/sec_certs/utils/extract.py
+++ b/src/sec_certs/utils/extract.py
@@ -741,29 +741,6 @@ def load_text_file(
 
 def rules_get_subset(desired_path: str) -> dict:
     """
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
     Recursively applies cc_certs.get(key) on tokens from desired_path,
     returns the keys of the inner-most layer.
     """
diff --git a/src/sec_certs/utils/helpers.py b/src/sec_certs/utils/helpers.py
index dedee959..6a6a9954 100644
--- a/src/sec_certs/utils/helpers.py
+++ b/src/sec_certs/utils/helpers.py
@@ -11,6 +11,7 @@ from functools import partial
 from pathlib import Path
 from typing import Any
 
+import dateparser
 import numpy as np
 import pkgconfig
 import requests
@@ -21,7 +22,6 @@ from sec_certs.utils.tqdm import tqdm
 
 logger = logging.getLogger(__name__)
 
-
 _PROXIES = {
     "https://www.commoncriteriaportal.org/": "https://sec-certs.org/proxy/cc/",
     "https://csrc.nist.gov/": "https://sec-certs.org/proxy/fips/",
@@ -120,6 +120,20 @@ def to_utc(timestamp: datetime) -> datetime:
     return timestamp.replace(tzinfo=None)
 
 
+def parse_date(date_string: str | None, format: str | None = None, languages: list[str] | None = None):
+    if not date_string:
+        return None
+    if format:
+        try:
+            return datetime.strptime(date_string, format).date()
+        except ValueError:
+            pass
+    parsed = dateparser.parse(date_string, languages=languages)
+    if parsed is not None:
+        return parsed.date()
+    return None
+
+
 def is_in_dict(target_dict: dict, path: str) -> bool:
     current_level = target_dict
     for item in path:
diff --git a/src/sec_certs/utils/sanitization.py b/src/sec_certs/utils/sanitization.py
index 4a7326eb..9bf4ad95 100644
--- a/src/sec_certs/utils/sanitization.py
+++ b/src/sec_certs/utils/sanitization.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import html
 import logging
+import re
 from datetime import date
 from pathlib import Path
 from urllib.parse import urlparse
@@ -16,7 +17,9 @@ logger = logging.getLogger(__name__)
 def sanitize_navigable_string(string: NavigableString | str | None) -> str | None:
     if not string:
         return None
-    return str(string).strip().replace("\xad", "").replace("\xa0", "")
+    rex = re.compile(r"\s+")
+    string = str(string).strip().replace("\xad", "").replace("\xa0", "")
+    return rex.sub(" ", string)
 
 
 def sanitize_link(record: str | None) -> str | None:
diff --git a/src/sec_certs/utils/tqdm.py b/src/sec_certs/utils/tqdm.py
index 581295ad..0a23903f 100644
--- a/src/sec_certs/utils/tqdm.py
+++ b/src/sec_certs/utils/tqdm.py
@@ -1,4 +1,4 @@
-from tqdm import tqdm as tqdm_original
+from tqdm.auto import tqdm as tqdm_original
 
 from sec_certs.configuration import config