diff options
Diffstat (limited to 'notebooks/cc/reference_annotations/data_preprocessing.ipynb')
| -rw-r--r-- | notebooks/cc/reference_annotations/data_preprocessing.ipynb | 42 |
1 files changed, 28 insertions, 14 deletions
diff --git a/notebooks/cc/reference_annotations/data_preprocessing.ipynb b/notebooks/cc/reference_annotations/data_preprocessing.ipynb index 76d16326..e5a55c71 100644 --- a/notebooks/cc/reference_annotations/data_preprocessing.ipynb +++ b/notebooks/cc/reference_annotations/data_preprocessing.ipynb @@ -29,6 +29,7 @@ "from sec_certs.utils.parallel_processing import process_parallel\n", "import pandas as pd\n", "import json\n", + "import langdetect\n", "\n", "nlp = spacy.load(\"en_core_web_sm\")\n", "from pathlib import Path\n", @@ -128,7 +129,13 @@ "\n", " return pd.concat(\n", " [load_single_df(train_path, \"train\"), load_single_df(valid_path, \"valid\"), load_single_df(test_path, \"test\")]\n", - " )[[\"dgst\", \"referenced_cert_id\", \"source\", \"label\", \"comment\"]]" + " )[[\"dgst\", \"referenced_cert_id\", \"source\", \"label\", \"comment\"]]\n", + "\n", + "def preprocess_sentence(x):\n", + " pass\n", + "\n", + "def preprocess_sentences(sentences: list[str]):\n", + " return [preprocess_sentence(x) for x in sentences]" ] }, { @@ -141,15 +148,15 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 49, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 944/944 [01:05<00:00, 14.48it/s]\n", - "100%|██████████| 2288/2288 [00:32<00:00, 69.50it/s]\n" + "100%|██████████| 944/944 [01:07<00:00, 14.06it/s]\n", + "100%|██████████| 2288/2288 [00:36<00:00, 62.14it/s]\n" ] } ], @@ -192,7 +199,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -208,19 +215,26 @@ " annotations_df[[\"dgst\", \"referenced_cert_id\", \"label\"]].set_index([\"dgst\", \"referenced_cert_id\"]).label.to_dict()\n", ")\n", "\n", - "# TODO: We should investigate the cases when we match no sentence, they may be new-lines and stuff\n", - "# TODO: Add language detection\n", - "# Process\n", + "# TODO: We should investigate the cases when df.sentences.isnull(), they may be new-lines and stuff\n", + "\n", + "# Enrich the data with language of sentences, so far discard all that's not en, fr, de.\n", + "# process then\n", "df = (\n", - " df.assign(\n", - " split=df.dgst.map(split_dct),\n", + " df.loc[df.sentences.notnull()]\n", + " .explode(\"sentences\")\n", + " .assign(lang = lambda df_: df_.sentences.map(langdetect.detect))\n", + " .loc[lambda df_: df_.lang.isin({\"en\", \"fr\", \"de\"})]\n", + " .groupby([\"dgst\", \"referenced_cert_id\", \"source\", \"label\"], as_index=False, dropna=False)\n", + " .agg({\"sentences\": list, \"lang\": list})\n", + " .assign(\n", + " split=lambda df_: df_.dgst.map(split_dct),\n", " label=lambda df_: [annotations_dict.get(x) for x in zip(df_[\"dgst\"], df_[\"referenced_cert_id\"])],\n", " )\n", - " .loc[lambda df_: (df_[\"sentences\"].notnull()) & (df_[\"split\"] != \"test\")]\n", - " .groupby([\"dgst\", \"referenced_cert_id\", \"label\", \"split\"], as_index=False, dropna=False)[\"sentences\"]\n", - " .agg({\"sentences\": lambda x: set.union(*x)})\n", + " .loc[lambda df_: df_[\"split\"] != \"test\"]\n", + " .groupby([\"dgst\", \"referenced_cert_id\", \"label\", \"split\"], as_index=False, dropna=False)\n", + " .agg({\"sentences\": sum, \"lang\": sum})\n", ")\n", - "df.to_csv(REPO_ROOT / \"datasets/reference_classification_dataset.csv\", index=False)" + "df.to_csv(REPO_ROOT / \"datasets/reference_classification_dataset.csv\", index=False)\n" ] } ], |