diff options
| author | adamjanovsky | 2023-09-29 13:20:13 +0200 |
|---|---|---|
| committer | adamjanovsky | 2023-09-29 13:20:13 +0200 |
| commit | ee00fb283d8c1685abdddcbf4e097ea6ca911f09 (patch) | |
| tree | 78f415e4f3817d320b122cb1fc7fafe215502110 /src/sec_certs/utils | |
| parent | f59e517490112c8baeb56c81a5869774ad1b821b (diff) | |
| download | sec-certs-ee00fb283d8c1685abdddcbf4e097ea6ca911f09.tar.gz sec-certs-ee00fb283d8c1685abdddcbf4e097ea6ca911f09.tar.zst sec-certs-ee00fb283d8c1685abdddcbf4e097ea6ca911f09.zip | |
ditch lang, fix groupby
Diffstat (limited to 'src/sec_certs/utils')
| -rw-r--r-- | src/sec_certs/utils/nlp.py | 6 |
1 files changed, 2 insertions, 4 deletions
diff --git a/src/sec_certs/utils/nlp.py b/src/sec_certs/utils/nlp.py index cfa4aaf9..1e8dc911 100644 --- a/src/sec_certs/utils/nlp.py +++ b/src/sec_certs/utils/nlp.py @@ -29,10 +29,8 @@ def filter_short_sentences(sentences, actual_reference_keywords): def prepare_reference_annotations_df(df: pd.DataFrame): if df.loc[(df.label != "SELF") & (df.label.notnull())].empty: raise ValueError("No expert annotations found in the dataset of references.") - df = ( - df.loc[lambda df_: (df_.label != "SELF") & (df_.label.notnull())] - .assign(segments=lambda df_: eval_strings_if_necessary(df_.segments)) - .drop(columns="lang") + df = df.loc[lambda df_: (df_.label != "SELF") & (df_.label.notnull())].assign( + segments=lambda df_: eval_strings_if_necessary(df_.segments) ) df.segments = df.apply( lambda row: filter_short_sentences(row["segments"], row["actual_reference_keywords"]), axis=1 |