fixes and bump reqs

6 files changed, 19 insertions, 72 deletions

diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index a3755f97..a0815502 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -103,8 +103,6 @@ docutils==0.19
     #   sphinx
 evaluate==0.4.0
     # via setfit
-exceptiongroup==1.1.1
-    # via pytest
 executing==1.2.0
     # via stack-data
 fastjsonschema==2.16.3
@@ -148,13 +146,7 @@ imagesize==1.4.1
 importlib-metadata==6.0.0
     # via
     #   jupyter-cache
-    #   jupyter-client
     #   myst-nb
-    #   sphinx
-importlib-resources==5.12.0
-    # via
-    #   jsonschema
-    #   matplotlib
 iniconfig==2.0.0
     # via pytest
 ipykernel==6.21.3
@@ -346,8 +338,6 @@ pip-tools==6.12.3
     # via sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 platformdirs==3.1.1
     # via
     #   black
@@ -419,9 +409,7 @@ python-dateutil==2.8.2
     #   pandas
     #   sec-certs (./../pyproject.toml)
 pytz==2022.7.1
-    # via
-    #   babel
-    #   pandas
+    # via pandas
 pyyaml==6.0
     # via
     #   datasets
@@ -551,15 +539,6 @@ threadpoolctl==3.1.0
     # via scikit-learn
 tokenizers==0.13.2
     # via transformers
-tomli==2.0.1
-    # via
-    #   black
-    #   build
-    #   coverage
-    #   mypy
-    #   pyproject-hooks
-    #   pytest
-    #   setuptools-scm
 torch==2.0.0
     # via
     #   sentence-transformers
@@ -607,13 +586,11 @@ types-urllib3==1.26.25.8
     # via types-requests
 typing-extensions==4.5.0
     # via
-    #   black
     #   huggingface-hub
     #   mypy
     #   myst-nb
     #   myst-parser
     #   pydantic
-    #   pypdf
     #   setuptools-scm
     #   torch
 urllib3==1.26.15
@@ -643,9 +620,7 @@ xxhash==3.2.0
 yarl==1.8.2
     # via aiohttp
 zipp==3.15.0
-    # via
-    #   importlib-metadata
-    #   importlib-resources
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index c1b2db28..dd9d3685 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -99,8 +99,6 @@ entrypoints==0.4
     # via jupyter-client
 evaluate==0.4.0
     # via setfit
-exceptiongroup==1.0.4
-    # via pytest
 executing==1.2.0
     # via stack-data
 fastjsonschema==2.16.2
@@ -145,9 +143,6 @@ importlib-metadata==5.1.0
     # via
     #   jupyter-cache
     #   myst-nb
-    #   sphinx
-importlib-resources==5.10.1
-    # via jsonschema
 iniconfig==1.1.1
     # via pytest
 ipykernel==6.19.1
@@ -340,8 +335,6 @@ pip-tools==6.11.0
     # via sec-certs (./../pyproject.toml)
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 platformdirs==2.6.0
     # via
     #   black
@@ -543,14 +536,7 @@ tokenizers==0.13.2
 toml==0.10.2
     # via pre-commit
 tomli==2.0.1
-    # via
-    #   black
-    #   build
-    #   coverage
-    #   mypy
-    #   pep517
-    #   pytest
-    #   setuptools-scm
+    # via setuptools-scm
 torch==2.0.1
     # via
     #   sentence-transformers
@@ -598,13 +584,11 @@ types-urllib3==1.26.25.4
     # via types-requests
 typing-extensions==4.4.0
     # via
-    #   black
     #   huggingface-hub
     #   mypy
     #   myst-nb
     #   myst-parser
     #   pydantic
-    #   pypdf
     #   setuptools-scm
     #   torch
 urllib3==1.26.13
@@ -634,9 +618,7 @@ xxhash==3.2.0
 yarl==1.8.2
     # via aiohttp
 zipp==3.11.0
-    # via
-    #   importlib-metadata
-    #   importlib-resources
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 98b45030..03a85889 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -102,8 +102,6 @@ idna==3.4
     # via
     #   requests
     #   yarl
-importlib-resources==5.10.1
-    # via jsonschema
 ipykernel==6.19.1
     # via
     #   ipywidgets
@@ -238,8 +236,6 @@ pillow==9.3.0
     #   torchvision
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 platformdirs==2.6.0
     # via jupyter-core
 preshed==3.0.8
@@ -408,7 +404,6 @@ typing-extensions==4.4.0
     # via
     #   huggingface-hub
     #   pydantic
-    #   pypdf
     #   setuptools-scm
     #   torch
 urllib3==1.26.13
@@ -431,8 +426,6 @@ xxhash==3.2.0
     #   evaluate
 yarl==1.9.2
     # via aiohttp
-zipp==3.11.0
-    # via importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 917d46a6..0981a4ea 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -77,8 +77,6 @@ entrypoints==0.4
     # via jupyter-client
 evaluate==0.4.0
     # via setfit
-exceptiongroup==1.0.4
-    # via pytest
 executing==1.2.0
     # via stack-data
 filelock==3.12.0
@@ -109,8 +107,6 @@ idna==3.4
     # via
     #   requests
     #   yarl
-importlib-resources==5.10.1
-    # via jsonschema
 iniconfig==1.1.1
     # via pytest
 ipykernel==6.19.1
@@ -248,8 +244,6 @@ pillow==9.3.0
     #   torchvision
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
-pkgutil-resolve-name==1.3.10
-    # via jsonschema
 platformdirs==2.6.0
     # via jupyter-core
 pluggy==1.0.0
@@ -386,10 +380,7 @@ threadpoolctl==3.1.0
 tokenizers==0.13.3
     # via transformers
 tomli==2.0.1
-    # via
-    #   coverage
-    #   pytest
-    #   setuptools-scm
+    # via setuptools-scm
 torch==2.0.1
     # via
     #   sentence-transformers
@@ -429,7 +420,6 @@ typing-extensions==4.4.0
     # via
     #   huggingface-hub
     #   pydantic
-    #   pypdf
     #   setuptools-scm
     #   torch
 urllib3==1.26.13
@@ -452,8 +442,6 @@ xxhash==3.2.0
     #   evaluate
 yarl==1.9.2
     # via aiohttp
-zipp==3.11.0
-    # via importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/src/sec_certs/configuration.py b/src/sec_certs/configuration.py
index 3e49ecad..1dda9d7c 100644
--- a/src/sec_certs/configuration.py
+++ b/src/sec_certs/configuration.py
@@ -5,7 +5,8 @@ from pathlib import Path
 from typing import Literal, Optional
 
 import yaml
-from pydantic import AnyHttpUrl, BaseSettings, Field
+from pydantic import AnyHttpUrl, Field
+from pydantic_settings import BaseSettings
 
 
 class Configuration(BaseSettings):
diff --git a/src/sec_certs/model/references/segment_extractor.py b/src/sec_certs/model/references/segment_extractor.py
index 8a01e840..bbb526b1 100644
--- a/src/sec_certs/model/references/segment_extractor.py
+++ b/src/sec_certs/model/references/segment_extractor.py
@@ -110,7 +110,7 @@ def preprocess_txt_func(data: str, actual_reference_keywords: frozenset[str]) ->
 
 
 def replace_citation_identifiers(data: str, actual_reference_keywords: frozenset[str]) -> str:
-    segments = {sent.text for sent in nlp(data).sents if any([x in sent.text for x in actual_reference_keywords])}
+    segments = {sent.text for sent in nlp(data).sents if any(x in sent.text for x in actual_reference_keywords)}
     patterns_to_replace = find_bracket_pattern(segments, actual_reference_keywords)
     for x in patterns_to_replace:
         data = data.replace(x[0], x[1])
@@ -274,10 +274,18 @@ class ReferenceSegmentExtractor:
 
         def load_single_df(pth: Path, split_name: str) -> pd.DataFrame:
             return (
-                pd.read_csv(pth)
-                .assign(label=lambda df_: df_.label.str.replace(" ", "_").str.upper(), split=split_name)
-                .replace("NONE", None)
+                pd.read_csv(
+                    pth,
+                    dtype={
+                        "dgst": str,
+                        "canonical_reference_keyword": str,
+                        "source": str,
+                        "label": str,
+                        "comment": str,
+                    },
+                )
                 .dropna(subset="label")
+                .assign(label=lambda df_: df_.label.str.replace(" ", "_").str.upper(), split=split_name)
             )
 
         annotations_directory = Path(str(files("sec_certs.data") / "reference_annotations/manual_annotations/"))