analysis of references to unknown certs

1 files changed, 45 insertions, 1 deletions

diff --git a/notebooks/cc/references.ipynb b/notebooks/cc/references.ipynb
index bb79f585..118bdf3b 100644
--- a/notebooks/cc/references.ipynb
+++ b/notebooks/cc/references.ipynb
@@ -20,7 +20,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 29,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -35,6 +35,8 @@
     "import matplotlib.pyplot as plt\n",
     "import matplotlib.ticker as mtick\n",
     "from sec_certs.dataset.cc import CCDataset\n",
+    "from sec_certs.model.references.reference_finder import ReferenceFinder\n",
+    "from sec_certs.sample.cc_certificate_id import CertificateId\n",
     "import pandas as pd\n",
     "import seaborn as sns\n",
     "import numpy as np\n",
@@ -123,6 +125,48 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "### References to certificate with unknown IDs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def ref_lookup(kw_attr):\n",
+    "            def func(cert):\n",
+    "                kws = getattr(cert.pdf_data, kw_attr)\n",
+    "                if not kws:\n",
+    "                    return set()\n",
+    "                res = set()\n",
+    "                for scheme, matches in kws[\"cc_cert_id\"].items():\n",
+    "                    for match in matches:\n",
+    "                        try:\n",
+    "                            canonical = CertificateId(scheme, match).canonical\n",
+    "                            res.add(canonical)\n",
+    "                        except Exception:\n",
+    "                            res.add(match)\n",
+    "                return res\n",
+    "\n",
+    "            return func\n",
+    "\n",
+    "report_finder = ReferenceFinder()\n",
+    "report_finder.fit(dset.certs, lambda cert: cert.heuristics.cert_id, ref_lookup(\"report_keywords\"))\n",
+    "report_unknown_references = report_finder.unknown_references\n",
+    "\n",
+    "st_finder = ReferenceFinder()\n",
+    "st_finder.fit(dset.certs, lambda cert: cert.heuristics.cert_id, ref_lookup(\"st_keywords\"))\n",
+    "st_unknown_references = st_finder.unknown_references\n",
+    "\n",
+    "# References are stored in `report_unknown_references` and `st_unknown_references` for further analysis"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
     "### Count numbers of reference-rich certificates\n",
     "\n",
     "- From the numbers follows that whenever a certificate is directly referencing some else, it also indirectly references some else\n",