diff options
| -rw-r--r-- | .github/workflows/lint.yml | 10 | ||||
| -rw-r--r-- | .github/workflows/tests.yml | 7 | ||||
| -rw-r--r-- | CONTRIBUTING.md | 9 | ||||
| -rw-r--r-- | dev_requirements.txt | 12 | ||||
| -rw-r--r-- | docker/Dockerfile | 2 | ||||
| -rw-r--r-- | requirements.txt | 43 | ||||
| -rwxr-xr-x | requirements/compile.sh | 7 | ||||
| -rw-r--r-- | requirements/dev_requirements.in | 13 | ||||
| -rw-r--r-- | requirements/dev_requirements.txt | 53 | ||||
| -rw-r--r-- | requirements/requirements.in | 21 | ||||
| -rw-r--r-- | requirements/requirements.txt | 43 | ||||
| -rw-r--r-- | requirements/test_requirements.in | 3 | ||||
| -rw-r--r-- | requirements/test_requirements.txt | 10 | ||||
| -rw-r--r-- | setup.py | 12 |
14 files changed, 170 insertions, 75 deletions
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d49b7224..1c9741da 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,13 +14,9 @@ jobs: - name: Install external dependencies run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y - name: Install python dependencies - run: pip install -r requirements.txt - - name: Install mypy and types run: | - pip install mypy - python3 -m pip install types-PyYAML - python3 -m pip install types-python-dateutil - python3 -m pip install types-requests + pip install -r requirements/requirements.txt + pip install -r requirements/dev_requirements.txt - name: Run mypy run: mypy . black: @@ -37,7 +33,7 @@ jobs: python-version: 3.8 - uses: isort/isort-action@master with: - requirementsFiles: "requirements.txt dev_requirements.txt" + requirementsFiles: "requirements/requirements.txt requirements/dev_requirements.txt" flake8-lint: runs-on: ubuntu-latest name: Flake8 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 84c8bc12..3cba8d28 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,6 @@ on: env: DEBIAN_FRONTEND: noninteractive - jobs: run-test: runs-on: ubuntu-latest @@ -24,11 +23,9 @@ jobs: with: python-version: '3.8' - name: Install python dependencies - run: pip install -r requirements.txt + run: pip install -r requirements/requirements.txt - name: Install pytest and package run: | - pip install pytest - pip install pytest-cov - pip install ".[dev,test]" + pip install -r requirements/test_requirements.txt - name: Run tests run: pytest --cov=sec_certs tests diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ce96222..d539f656 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,14 @@ Our [Dockerfile](https://github.com/crocs-muni/sec-certs/blob/main/docker/Docker - Some imported libraries have non-trivial dependencies to resolve: - [pdftotext](https://github.com/jalan/pdftotext) requires [Poppler](https://poppler.freedesktop.org/) to be installed. We've experienced issues with older versions of Poppler (`0.x`), make sure to install `20.x` version of these libraries. - [graphviz](https://pypi.org/project/graphviz/) requires `graphviz` to be on the path -- + +### Requirements + +Requirements are maintained with [pip-tools](https://github.com/jazzband/pip-tools). The main ideas are: +- List actual dependencies in `.in` files inside [requirements](requirements) folder without pinning them. +- Those dependencies are loaded into [setup.py](setup.py) file. +- Additionally, [compile.sh](requirements/compile.sh) script is used to compile pinned versions of requirements that reside in `.txt` files in the same folder. +- Tests, linting and Docker all run against this reproducible environment of pinned requirements. ## Branches and releases diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index f2cceafd..00000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -mypy -types-PyYAML -types-python-dateutil -types-requests -pytest==6.2.5 -pytest-cov==3.0.0 -pytest-monitor==1.6.3 -pytest-profiling==1.7.0 -black -isort==5.10.1 -flake8==4.0.1 -pre-commit
\ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index c691716c..2c138d46 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,7 +42,7 @@ ENV PATH="${VENV_PATH}/bin:$PATH" RUN \ pip3 install -U pip && \ pip3 install wheel && \ - pip3 install -r requirements.txt && \ + pip3 install -r requirements/requirements.txt && \ pip3 install --no-cache notebook # #just to be sure that pdftotext is in $PATH diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4d66b8e7..00000000 --- a/requirements.txt +++ /dev/null @@ -1,43 +0,0 @@ -attrs==21.4.0 -beautifulsoup4==4.10.0 -billiard==3.6.4.0 -certifi==2021.10.8 -chardet==4.0.0 -charset-normalizer==2.0.9 -click==8.0.3 -cycler==0.11.0 -distro==1.6.0 -fonttools==4.28.5 -graphviz==0.19.1 -html5lib==1.1 -idna==3.3 -importlib-resources==5.4.0 -joblib==1.1.0 -jsonschema==4.3.3 -kiwisolver==1.3.2 -lxml==4.7.1 -matplotlib==3.5.1 -numpy==1.22.0 -packaging==21.3 -pandas==1.3.5 -pikepdf==4.3.1 -Pillow==9.0.1 -pyparsing==3.0.6 -PyPDF2==1.26.0 -pyrsistent==0.18.0 -python-dateutil==2.8.2 -pytz==2021.3 -PyYAML==6.0 -rapidfuzz==1.9.1 -requests==2.27.0 -scikit-learn==1.0.2 -scipy==1.7.3 -six==1.16.0 -soupsieve==2.3.1 -tabula-py==2.3.0 -tabulate==0.8.9 -threadpoolctl==3.0.0 -tqdm==4.62.3 -urllib3==1.26.7 -webencodings==0.5.1 -pdftotext==2.2.2 diff --git a/requirements/compile.sh b/requirements/compile.sh new file mode 100755 index 00000000..4d077764 --- /dev/null +++ b/requirements/compile.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# See CONTRIBUTING.md for description + +pip-compile requirements.in --no-header --annotation-style line +pip-compile dev_requirements.in --no-header --annotation-style line +pip-compile test_requirements.in --no-header --annotation-style line diff --git a/requirements/dev_requirements.in b/requirements/dev_requirements.in new file mode 100644 index 00000000..54bba20d --- /dev/null +++ b/requirements/dev_requirements.in @@ -0,0 +1,13 @@ +mypy +types-PyYAML +types-python-dateutil +types-requests +pytest +pytest-cov +pytest-monitor +pytest-profiling +black +isort +flake8 +pre-commit +pip-tools
\ No newline at end of file diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt new file mode 100644 index 00000000..d11d2459 --- /dev/null +++ b/requirements/dev_requirements.txt @@ -0,0 +1,53 @@ +attrs==21.4.0 # via pytest +black==22.3.0 # via -r dev_requirements.in +certifi==2021.10.8 # via requests +cfgv==3.3.1 # via pre-commit +charset-normalizer==2.0.12 # via requests +click==8.1.2 # via black, pip-tools +coverage[toml]==6.3.2 # via pytest-cov +distlib==0.3.4 # via virtualenv +filelock==3.6.0 # via virtualenv +flake8==4.0.1 # via -r dev_requirements.in +gprof2dot==2021.2.21 # via pytest-profiling +identify==2.4.12 # via pre-commit +idna==3.3 # via requests +iniconfig==1.1.1 # via pytest +isort==5.10.1 # via -r dev_requirements.in +mccabe==0.6.1 # via flake8 +memory-profiler==0.60.0 # via pytest-monitor +mypy==0.942 # via -r dev_requirements.in +mypy-extensions==0.4.3 # via black, mypy +nodeenv==1.6.0 # via pre-commit +packaging==21.3 # via pytest +pathspec==0.9.0 # via black +pep517==0.12.0 # via pip-tools +pip-tools==6.5.1 # via -r dev_requirements.in +platformdirs==2.5.1 # via black, virtualenv +pluggy==1.0.0 # via pytest +pre-commit==2.18.1 # via -r dev_requirements.in +psutil==5.9.0 # via memory-profiler, pytest-monitor +py==1.11.0 # via pytest +pycodestyle==2.8.0 # via flake8 +pyflakes==2.4.0 # via flake8 +pyparsing==3.0.7 # via packaging +pytest==7.1.1 # via -r dev_requirements.in, pytest-cov, pytest-monitor, pytest-profiling +pytest-cov==3.0.0 # via -r dev_requirements.in +pytest-monitor==1.6.3 # via -r dev_requirements.in +pytest-profiling==1.7.0 # via -r dev_requirements.in +pyyaml==6.0 # via pre-commit +requests==2.27.1 # via pytest-monitor +six==1.16.0 # via pytest-profiling, virtualenv +toml==0.10.2 # via pre-commit +tomli==2.0.1 # via black, coverage, mypy, pep517, pytest +types-python-dateutil==2.8.10 # via -r dev_requirements.in +types-pyyaml==6.0.5 # via -r dev_requirements.in +types-requests==2.27.16 # via -r dev_requirements.in +types-urllib3==1.26.11 # via types-requests +typing-extensions==4.1.1 # via black, mypy +urllib3==1.26.9 # via requests +virtualenv==20.14.0 # via pre-commit +wheel==0.37.1 # via pip-tools, pytest-monitor + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements/requirements.in b/requirements/requirements.in new file mode 100644 index 00000000..f7fc03a3 --- /dev/null +++ b/requirements/requirements.in @@ -0,0 +1,21 @@ +beautifulsoup4 +billiard +click +graphviz +html5lib +jsonschema +lxml +matplotlib +numpy +pandas +pdftotext +pikepdf +Pillow +PyPDF2 +python-dateutil +PyYAML +rapidfuzz +requests +scikit-learn +tabula-py +tqdm diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 00000000..7be04c47 --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,43 @@ +attrs==21.4.0 # via jsonschema +beautifulsoup4==4.10.0 # via -r requirements.in +billiard==3.6.4.0 # via -r requirements.in +certifi==2021.10.8 # via requests +charset-normalizer==2.0.12 # via requests +click==8.1.2 # via -r requirements.in +cycler==0.11.0 # via matplotlib +distro==1.7.0 # via tabula-py +fonttools==4.31.2 # via matplotlib +graphviz==0.19.1 # via -r requirements.in +html5lib==1.1 # via -r requirements.in +idna==3.3 # via requests +importlib-resources==5.6.0 # via jsonschema +jarowinkler==1.0.2 # via rapidfuzz +joblib==1.1.0 # via scikit-learn +jsonschema==4.4.0 # via -r requirements.in +kiwisolver==1.4.2 # via matplotlib +lxml==4.8.0 # via -r requirements.in, pikepdf +matplotlib==3.5.1 # via -r requirements.in +numpy==1.22.3 # via -r requirements.in, matplotlib, pandas, scikit-learn, scipy, tabula-py +packaging==21.3 # via matplotlib, pikepdf +pandas==1.4.2 # via -r requirements.in, tabula-py +pdftotext==2.2.2 # via -r requirements.in +pikepdf==5.1.1 # via -r requirements.in +pillow==9.1.0 # via -r requirements.in, matplotlib, pikepdf +pyparsing==3.0.7 # via matplotlib, packaging +pypdf2==1.26.0 # via -r requirements.in +pyrsistent==0.18.1 # via jsonschema +python-dateutil==2.8.2 # via -r requirements.in, matplotlib, pandas +pytz==2022.1 # via pandas +pyyaml==6.0 # via -r requirements.in +rapidfuzz==2.0.7 # via -r requirements.in +requests==2.27.1 # via -r requirements.in +scikit-learn==1.0.2 # via -r requirements.in +scipy==1.8.0 # via scikit-learn +six==1.16.0 # via html5lib, python-dateutil +soupsieve==2.3.1 # via beautifulsoup4 +tabula-py==2.3.0 # via -r requirements.in +threadpoolctl==3.1.0 # via scikit-learn +tqdm==4.64.0 # via -r requirements.in +urllib3==1.26.9 # via requests +webencodings==0.5.1 # via html5lib +zipp==3.8.0 # via importlib-resources diff --git a/requirements/test_requirements.in b/requirements/test_requirements.in new file mode 100644 index 00000000..0c69b775 --- /dev/null +++ b/requirements/test_requirements.in @@ -0,0 +1,3 @@ +pytest +coverage +pytest-cov diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt new file mode 100644 index 00000000..d47239ea --- /dev/null +++ b/requirements/test_requirements.txt @@ -0,0 +1,10 @@ +attrs==21.4.0 # via pytest +coverage[toml]==6.3.2 # via -r test_requirements.in, pytest-cov +iniconfig==1.1.1 # via pytest +packaging==21.3 # via pytest +pluggy==1.0.0 # via pytest +py==1.11.0 # via pytest +pyparsing==3.0.7 # via packaging +pytest==7.1.1 # via -r test_requirements.in, pytest-cov +pytest-cov==3.0.0 # via -r test_requirements.in +tomli==2.0.1 # via coverage, pytest @@ -1,12 +1,9 @@ #!/usr/bin/env python3 from setuptools import find_packages, setup -with open("requirements.txt") as f: - requirements = f.read().splitlines() - setup( name="sec-certs", - author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky", + author="Petr Svenda, Stanislav Bobon, Jan Jancar, Adam Janovsky, Jiri Michalik", author_email="svenda@fi.muni.cz", version_config=True, setup_requires=["setuptools-git-versioning"], @@ -27,8 +24,11 @@ setup( "Intended Audience :: Science/Research", ], python_requires=">=3.8", - install_requires=requirements, - extras_require={"dev": ["mypy", "flake8"], "test": ["pytest", "coverage"]}, + install_requires=open("requirements/requirements.in", "r").read().splitlines(), + extras_require={ + "dev": open("requirements/dev_requirements.in", "r").read().splitlines(), + "test": open("requirements/test_requirements.in", "r").read().splitlines(), + }, include_package_data=True, package_data={"sec_certs": ["settings.yaml", "settings-schema.json"]}, entry_points={"console_scripts": ["cc-certs=cc_cli:main", "fips-certs=fips_cli:main"]}, |