From ee00fb283d8c1685abdddcbf4e097ea6ca911f09 Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Fri, 29 Sep 2023 13:20:13 +0200 Subject: ditch lang, fix groupby --- src/sec_certs/utils/nlp.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/sec_certs/utils') diff --git a/src/sec_certs/utils/nlp.py b/src/sec_certs/utils/nlp.py index cfa4aaf9..1e8dc911 100644 --- a/src/sec_certs/utils/nlp.py +++ b/src/sec_certs/utils/nlp.py @@ -29,10 +29,8 @@ def filter_short_sentences(sentences, actual_reference_keywords): def prepare_reference_annotations_df(df: pd.DataFrame): if df.loc[(df.label != "SELF") & (df.label.notnull())].empty: raise ValueError("No expert annotations found in the dataset of references.") - df = ( - df.loc[lambda df_: (df_.label != "SELF") & (df_.label.notnull())] - .assign(segments=lambda df_: eval_strings_if_necessary(df_.segments)) - .drop(columns="lang") + df = df.loc[lambda df_: (df_.label != "SELF") & (df_.label.notnull())].assign( + segments=lambda df_: eval_strings_if_necessary(df_.segments) ) df.segments = df.apply( lambda row: filter_short_sentences(row["segments"], row["actual_reference_keywords"]), axis=1 -- cgit v1.3.1