setfit to regular reqs. to allow for tests

3 files changed, 294 insertions, 19 deletions

diff --git a/pyproject.toml b/pyproject.toml
index 08fd0f5b..938cf8aa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,6 +58,8 @@
     "scipy>=1.9.0",
     "networkx",
     "pydantic",
+    "setfit",
+    "langdetect",
   ]
 
   [project.optional-dependencies]
@@ -81,8 +83,6 @@
       "sphinx-design",
       "sphinx-copybutton",
       "ipython!=8.7.0",
-      "setfit",
-      "langdetect",
     ]
     test = ["pytest", "coverage", "pytest-cov"]
 
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index 1b769bfe..13a9cc3a 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -1,11 +1,21 @@
+aiohttp==3.8.4
+    # via
+    #   datasets
+    #   fsspec
+aiosignal==1.3.1
+    # via aiohttp
 appnope==0.1.3
     # via
     #   ipykernel
     #   ipython
 asttokens==2.2.1
     # via stack-data
+async-timeout==4.0.2
+    # via aiohttp
 attrs==22.1.0
-    # via jsonschema
+    # via
+    #   aiohttp
+    #   jsonschema
 backcall==0.2.0
     # via ipython
 beautifulsoup4==4.11.1
@@ -20,9 +30,12 @@ catalogue==2.0.8
 certifi==2022.12.7
     # via requests
 charset-normalizer==2.1.1
-    # via requests
+    # via
+    #   aiohttp
+    #   requests
 click==8.1.3
     # via
+    #   nltk
     #   sec-certs (./../pyproject.toml)
     #   typer
 comm==0.1.2
@@ -38,24 +51,57 @@ cymem==2.0.7
     #   preshed
     #   spacy
     #   thinc
+datasets==2.12.0
+    # via
+    #   evaluate
+    #   setfit
 debugpy==1.6.4
     # via ipykernel
 decorator==5.1.1
     # via ipython
 deprecation==2.1.0
     # via pikepdf
+dill==0.3.6
+    # via
+    #   datasets
+    #   evaluate
+    #   multiprocess
 distro==1.8.0
     # via tabula-py
 entrypoints==0.4
     # via jupyter-client
+evaluate==0.4.0
+    # via setfit
 executing==1.2.0
     # via stack-data
+filelock==3.12.0
+    # via
+    #   huggingface-hub
+    #   torch
+    #   transformers
 fonttools==4.38.0
     # via matplotlib
+frozenlist==1.3.3
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2023.5.0
+    # via
+    #   datasets
+    #   evaluate
+    #   huggingface-hub
 html5lib==1.1
     # via sec-certs (./../pyproject.toml)
+huggingface-hub==0.15.1
+    # via
+    #   datasets
+    #   evaluate
+    #   sentence-transformers
+    #   transformers
 idna==3.4
-    # via requests
+    # via
+    #   requests
+    #   yarl
 importlib-resources==5.10.1
     # via jsonschema
 ipykernel==6.19.1
@@ -71,9 +117,13 @@ ipywidgets==8.0.3
 jedi==0.18.2
     # via ipython
 jinja2==3.1.2
-    # via spacy
+    # via
+    #   spacy
+    #   torch
 joblib==1.2.0
-    # via scikit-learn
+    # via
+    #   nltk
+    #   scikit-learn
 jsonschema==4.17.3
     # via sec-certs (./../pyproject.toml)
 jupyter-client==7.4.8
@@ -86,6 +136,8 @@ kiwisolver==1.4.4
     # via matplotlib
 langcodes==3.3.0
     # via spacy
+langdetect==1.0.9
+    # via sec-certs (./../pyproject.toml)
 lxml==4.9.1
     # via
     #   pikepdf
@@ -101,6 +153,16 @@ matplotlib-inline==0.1.6
     # via
     #   ipykernel
     #   ipython
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.14
+    # via
+    #   datasets
+    #   evaluate
 murmurhash==1.0.9
     # via
     #   preshed
@@ -111,31 +173,47 @@ nest-asyncio==1.5.6
     #   ipykernel
     #   jupyter-client
 networkx==2.8.8
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   sec-certs (./../pyproject.toml)
+    #   torch
+nltk==3.8.1
+    # via sentence-transformers
 numpy==1.23.5
     # via
     #   blis
     #   contourpy
+    #   datasets
+    #   evaluate
     #   matplotlib
     #   pandas
+    #   pyarrow
     #   pysankeybeta
     #   scikit-learn
     #   scipy
     #   seaborn
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
     #   spacy
     #   tabula-py
     #   thinc
+    #   torchvision
+    #   transformers
 packaging==22.0
     # via
+    #   datasets
     #   deprecation
+    #   evaluate
+    #   huggingface-hub
     #   ipykernel
     #   matplotlib
     #   pikepdf
     #   setuptools-scm
     #   spacy
+    #   transformers
 pandas==1.5.2
     # via
+    #   datasets
+    #   evaluate
     #   pysankeybeta
     #   seaborn
     #   sec-certs (./../pyproject.toml)
@@ -157,6 +235,7 @@ pillow==9.3.0
     #   matplotlib
     #   pikepdf
     #   sec-certs (./../pyproject.toml)
+    #   torchvision
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
 pkgutil-resolve-name==1.3.10
@@ -175,6 +254,8 @@ ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
     # via stack-data
+pyarrow==12.0.0
+    # via datasets
 pycryptodome==3.16.0
     # via pypdf
 pydantic==1.10.2
@@ -202,33 +283,62 @@ python-dateutil==2.8.2
 pytz==2022.6
     # via pandas
 pyyaml==6.0
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   sec-certs (./../pyproject.toml)
+    #   transformers
 pyzmq==24.0.1
     # via
     #   ipykernel
     #   jupyter-client
 rapidfuzz==2.13.3
     # via sec-certs (./../pyproject.toml)
+regex==2023.6.3
+    # via
+    #   nltk
+    #   transformers
 requests==2.31.0
     # via
+    #   datasets
+    #   evaluate
+    #   fsspec
+    #   huggingface-hub
+    #   responses
     #   sec-certs (./../pyproject.toml)
     #   spacy
+    #   torchvision
+    #   transformers
+responses==0.18.0
+    # via
+    #   datasets
+    #   evaluate
 scikit-learn==1.2.0
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
 scipy==1.9.3
     # via
     #   scikit-learn
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
 seaborn==0.12.1
     # via
     #   pysankeybeta
     #   sec-certs (./../pyproject.toml)
+sentence-transformers==2.2.2
+    # via setfit
+sentencepiece==0.1.99
+    # via sentence-transformers
+setfit==0.7.0
+    # via sec-certs (./../pyproject.toml)
 setuptools-scm==7.0.5
     # via sec-certs (./../pyproject.toml)
 six==1.16.0
     # via
     #   asttokens
     #   html5lib
+    #   langdetect
     #   python-dateutil
 smart-open==6.2.0
     # via pathy
@@ -247,22 +357,38 @@ srsly==2.4.5
     #   thinc
 stack-data==0.6.2
     # via ipython
+sympy==1.12
+    # via torch
 tabula-py==2.6.0
     # via sec-certs (./../pyproject.toml)
 thinc==8.1.5
     # via spacy
 threadpoolctl==3.1.0
     # via scikit-learn
+tokenizers==0.13.3
+    # via transformers
 tomli==2.0.1
     # via setuptools-scm
+torch==2.0.1
+    # via
+    #   sentence-transformers
+    #   torchvision
+torchvision==0.15.2
+    # via sentence-transformers
 tornado==6.3.2
     # via
     #   ipykernel
     #   jupyter-client
 tqdm==4.64.1
     # via
+    #   datasets
+    #   evaluate
+    #   huggingface-hub
+    #   nltk
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
     #   spacy
+    #   transformers
 traitlets==5.6.0
     # via
     #   comm
@@ -272,17 +398,23 @@ traitlets==5.6.0
     #   jupyter-client
     #   jupyter-core
     #   matplotlib-inline
+transformers==4.29.2
+    # via sentence-transformers
 typer==0.7.0
     # via
     #   pathy
     #   spacy
 typing-extensions==4.4.0
     # via
+    #   huggingface-hub
     #   pydantic
     #   pypdf
     #   setuptools-scm
+    #   torch
 urllib3==1.26.13
-    # via requests
+    # via
+    #   requests
+    #   responses
 wasabi==0.10.1
     # via
     #   spacy
@@ -293,6 +425,12 @@ webencodings==0.5.1
     # via html5lib
 widgetsnbextension==4.0.4
     # via ipywidgets
+xxhash==3.2.0
+    # via
+    #   datasets
+    #   evaluate
+yarl==1.9.2
+    # via aiohttp
 zipp==3.11.0
     # via importlib-resources
 
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 70aab728..692d7e55 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -1,11 +1,20 @@
+aiohttp==3.8.4
+    # via
+    #   datasets
+    #   fsspec
+aiosignal==1.3.1
+    # via aiohttp
 appnope==0.1.3
     # via
     #   ipykernel
     #   ipython
 asttokens==2.2.1
     # via stack-data
+async-timeout==4.0.2
+    # via aiohttp
 attrs==22.1.0
     # via
+    #   aiohttp
     #   jsonschema
     #   pytest
 backcall==0.2.0
@@ -22,9 +31,12 @@ catalogue==2.0.8
 certifi==2022.12.7
     # via requests
 charset-normalizer==2.1.1
-    # via requests
+    # via
+    #   aiohttp
+    #   requests
 click==8.1.3
     # via
+    #   nltk
     #   sec-certs (./../pyproject.toml)
     #   typer
 comm==0.1.2
@@ -44,26 +56,59 @@ cymem==2.0.7
     #   preshed
     #   spacy
     #   thinc
+datasets==2.12.0
+    # via
+    #   evaluate
+    #   setfit
 debugpy==1.6.4
     # via ipykernel
 decorator==5.1.1
     # via ipython
 deprecation==2.1.0
     # via pikepdf
+dill==0.3.6
+    # via
+    #   datasets
+    #   evaluate
+    #   multiprocess
 distro==1.8.0
     # via tabula-py
 entrypoints==0.4
     # via jupyter-client
+evaluate==0.4.0
+    # via setfit
 exceptiongroup==1.0.4
     # via pytest
 executing==1.2.0
     # via stack-data
+filelock==3.12.0
+    # via
+    #   huggingface-hub
+    #   torch
+    #   transformers
 fonttools==4.38.0
     # via matplotlib
+frozenlist==1.3.3
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec[http]==2023.5.0
+    # via
+    #   datasets
+    #   evaluate
+    #   huggingface-hub
 html5lib==1.1
     # via sec-certs (./../pyproject.toml)
+huggingface-hub==0.15.1
+    # via
+    #   datasets
+    #   evaluate
+    #   sentence-transformers
+    #   transformers
 idna==3.4
-    # via requests
+    # via
+    #   requests
+    #   yarl
 importlib-resources==5.10.1
     # via jsonschema
 iniconfig==1.1.1
@@ -81,9 +126,13 @@ ipywidgets==8.0.3
 jedi==0.18.2
     # via ipython
 jinja2==3.1.2
-    # via spacy
+    # via
+    #   spacy
+    #   torch
 joblib==1.2.0
-    # via scikit-learn
+    # via
+    #   nltk
+    #   scikit-learn
 jsonschema==4.17.3
     # via sec-certs (./../pyproject.toml)
 jupyter-client==7.4.8
@@ -96,6 +145,8 @@ kiwisolver==1.4.4
     # via matplotlib
 langcodes==3.3.0
     # via spacy
+langdetect==1.0.9
+    # via sec-certs (./../pyproject.toml)
 lxml==4.9.1
     # via
     #   pikepdf
@@ -111,6 +162,16 @@ matplotlib-inline==0.1.6
     # via
     #   ipykernel
     #   ipython
+mpmath==1.3.0
+    # via sympy
+multidict==6.0.4
+    # via
+    #   aiohttp
+    #   yarl
+multiprocess==0.70.14
+    # via
+    #   datasets
+    #   evaluate
 murmurhash==1.0.9
     # via
     #   preshed
@@ -121,32 +182,48 @@ nest-asyncio==1.5.6
     #   ipykernel
     #   jupyter-client
 networkx==2.8.8
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   sec-certs (./../pyproject.toml)
+    #   torch
+nltk==3.8.1
+    # via sentence-transformers
 numpy==1.23.5
     # via
     #   blis
     #   contourpy
+    #   datasets
+    #   evaluate
     #   matplotlib
     #   pandas
+    #   pyarrow
     #   pysankeybeta
     #   scikit-learn
     #   scipy
     #   seaborn
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
     #   spacy
     #   tabula-py
     #   thinc
+    #   torchvision
+    #   transformers
 packaging==22.0
     # via
+    #   datasets
     #   deprecation
+    #   evaluate
+    #   huggingface-hub
     #   ipykernel
     #   matplotlib
     #   pikepdf
     #   pytest
     #   setuptools-scm
     #   spacy
+    #   transformers
 pandas==1.5.2
     # via
+    #   datasets
+    #   evaluate
     #   pysankeybeta
     #   seaborn
     #   sec-certs (./../pyproject.toml)
@@ -168,6 +245,7 @@ pillow==9.3.0
     #   matplotlib
     #   pikepdf
     #   sec-certs (./../pyproject.toml)
+    #   torchvision
 pkgconfig==1.5.5
     # via sec-certs (./../pyproject.toml)
 pkgutil-resolve-name==1.3.10
@@ -188,6 +266,8 @@ ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
     # via stack-data
+pyarrow==12.0.0
+    # via datasets
 pycryptodome==3.16.0
     # via pypdf
 pydantic==1.10.2
@@ -221,33 +301,62 @@ python-dateutil==2.8.2
 pytz==2022.6
     # via pandas
 pyyaml==6.0
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   datasets
+    #   huggingface-hub
+    #   sec-certs (./../pyproject.toml)
+    #   transformers
 pyzmq==24.0.1
     # via
     #   ipykernel
     #   jupyter-client
 rapidfuzz==2.13.3
     # via sec-certs (./../pyproject.toml)
+regex==2023.6.3
+    # via
+    #   nltk
+    #   transformers
 requests==2.31.0
     # via
+    #   datasets
+    #   evaluate
+    #   fsspec
+    #   huggingface-hub
+    #   responses
     #   sec-certs (./../pyproject.toml)
     #   spacy
+    #   torchvision
+    #   transformers
+responses==0.18.0
+    # via
+    #   datasets
+    #   evaluate
 scikit-learn==1.2.0
-    # via sec-certs (./../pyproject.toml)
+    # via
+    #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
 scipy==1.9.3
     # via
     #   scikit-learn
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
 seaborn==0.12.1
     # via
     #   pysankeybeta
     #   sec-certs (./../pyproject.toml)
+sentence-transformers==2.2.2
+    # via setfit
+sentencepiece==0.1.99
+    # via sentence-transformers
+setfit==0.7.0
+    # via sec-certs (./../pyproject.toml)
 setuptools-scm==7.0.5
     # via sec-certs (./../pyproject.toml)
 six==1.16.0
     # via
     #   asttokens
     #   html5lib
+    #   langdetect
     #   python-dateutil
 smart-open==6.2.0
     # via pathy
@@ -266,25 +375,41 @@ srsly==2.4.5
     #   thinc
 stack-data==0.6.2
     # via ipython
+sympy==1.12
+    # via torch
 tabula-py==2.6.0
     # via sec-certs (./../pyproject.toml)
 thinc==8.1.5
     # via spacy
 threadpoolctl==3.1.0
     # via scikit-learn
+tokenizers==0.13.3
+    # via transformers
 tomli==2.0.1
     # via
     #   coverage
     #   pytest
     #   setuptools-scm
+torch==2.0.1
+    # via
+    #   sentence-transformers
+    #   torchvision
+torchvision==0.15.2
+    # via sentence-transformers
 tornado==6.3.2
     # via
     #   ipykernel
     #   jupyter-client
 tqdm==4.64.1
     # via
+    #   datasets
+    #   evaluate
+    #   huggingface-hub
+    #   nltk
     #   sec-certs (./../pyproject.toml)
+    #   sentence-transformers
     #   spacy
+    #   transformers
 traitlets==5.6.0
     # via
     #   comm
@@ -294,17 +419,23 @@ traitlets==5.6.0
     #   jupyter-client
     #   jupyter-core
     #   matplotlib-inline
+transformers==4.29.2
+    # via sentence-transformers
 typer==0.7.0
     # via
     #   pathy
     #   spacy
 typing-extensions==4.4.0
     # via
+    #   huggingface-hub
     #   pydantic
     #   pypdf
     #   setuptools-scm
+    #   torch
 urllib3==1.26.13
-    # via requests
+    # via
+    #   requests
+    #   responses
 wasabi==0.10.1
     # via
     #   spacy
@@ -315,6 +446,12 @@ webencodings==0.5.1
     # via html5lib
 widgetsnbextension==4.0.4
     # via ipywidgets
+xxhash==3.2.0
+    # via
+    #   datasets
+    #   evaluate
+yarl==1.9.2
+    # via aiohttp
 zipp==3.11.0
     # via importlib-resources