fix error in ref. annot data preprocessing

1 files changed, 12 insertions, 63 deletions

diff --git a/notebooks/cc/reference_annotations/data_preprocessing.ipynb b/notebooks/cc/reference_annotations/data_preprocessing.ipynb
index 8c8edd31..5ee7dce5 100644
--- a/notebooks/cc/reference_annotations/data_preprocessing.ipynb
+++ b/notebooks/cc/reference_annotations/data_preprocessing.ipynb
@@ -16,15 +16,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 85,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
     "from __future__ import annotations\n",
     "\n",
     "from dataclasses import dataclass\n",
-    "from sec_certs.dataset import CCDataset\n",
     "from sec_certs.sample import CCCertificate\n",
+    "from sec_certs.dataset import CCDataset\n",
     "import spacy\n",
     "from sec_certs.utils.parallel_processing import process_parallel\n",
     "import pandas as pd\n",
@@ -98,17 +98,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 111,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 58/58 [00:07<00:00,  8.26it/s]\n",
-      "100%|██████████| 944/944 [01:08<00:00, 13.84it/s]\n",
-      "100%|██████████| 2259/2259 [00:33<00:00, 68.33it/s]\n",
-      "100%|██████████| 2551/2551 [00:02<00:00, 1089.81it/s]\n"
+      "100%|██████████| 58/58 [00:07<00:00,  8.27it/s]\n",
+      "100%|██████████| 944/944 [01:07<00:00, 14.02it/s]\n",
+      "100%|██████████| 2259/2259 [00:33<00:00, 68.05it/s]\n",
+      "100%|██████████| 2551/2551 [00:02<00:00, 1053.41it/s]\n"
      ]
     }
    ],
@@ -183,10 +183,10 @@
     "    )\n",
     "\n",
     "# With no label noise, we should be safe to fill in labels for sentences found in targets such that the corresponding report was annotated\n",
-    "df.label = df.copy().apply(\n",
-    "    lambda row: dgst_cert_id_to_label_mapping.get(\n",
-    "        (row[\"dgst\"], row[\"cert_id\"]) if pd.isnull(row[\"label\"]) else row[\"label\"]\n",
-    "    ),\n",
+    "df.label = df.apply(\n",
+    "    lambda row: dgst_cert_id_to_label_mapping.get((row[\"dgst\"], row[\"cert_id\"]))\n",
+    "    if pd.isnull(row[\"label\"])\n",
+    "    else row[\"label\"],\n",
     "    axis=1,\n",
     ")\n"
    ]
@@ -198,15 +198,12 @@
    "source": [
     "## Process Dataframes and dump to csv\n",
     "\n",
-    "1. Version with `dgst, cert_id, location, single_sentence` as `*_exploded.csv`\n",
-    "2. Version where all sentences tied to `(dgst, cert_id)` key are merged into `sentences`. Saved as `*_grouped.csv`\n",
-    "\n",
     "*Note*: So far don't work with test dataset"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 112,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -231,54 +228,6 @@
     "df = df.groupby([\"dgst\", \"cert_id\", \"label\", \"split\"], as_index=False, dropna=False)[\"sentences\"].agg({\"sentences\": lambda x: set.union(*x)})\n",
     "df.to_csv(REPO_ROOT / \"datasets/reference_classification_dataset.csv\", sep=';', index=False)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 113,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "Empty DataFrame\n",
-       "Columns: []\n",
-       "Index: []"
-      ]
-     },
-     "execution_count": 113,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "duplicate_df"
-   ]
   }
  ],
  "metadata": {