aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authoradamjanovsky2023-11-10 12:54:00 +0100
committerGitHub2023-11-10 12:54:00 +0100
commit1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e (patch)
treec0c2f35f921f3551f2bf0a146c1cbd6c0494f7a9
parent5968e53d9f78d744c1bd151be0d754282e65baee (diff)
parent10aa312c5e83c197a2d41fec710c5301bfff169e (diff)
downloadsec-certs-1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e.tar.gz
sec-certs-1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e.tar.zst
sec-certs-1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e.zip
Merge pull request #373 from crocs-muni/switch-black-to-ruff
switch black -> ruff format
-rw-r--r--.github/workflows/lint.yml44
-rw-r--r--.github/workflows/pre-commit.yml18
-rw-r--r--.pre-commit-config.yaml14
-rw-r--r--CONTRIBUTING.md4
-rw-r--r--notebooks/cc/cert_id_eval.ipynb48
-rw-r--r--pyproject.toml26
-rw-r--r--requirements/all_requirements.txt20
-rw-r--r--requirements/dev_requirements.txt22
-rw-r--r--requirements/requirements.txt4
-rw-r--r--requirements/test_requirements.txt4
-rw-r--r--src/sec_certs/dataset/dataset.py4
-rw-r--r--src/sec_certs/dataset/fips_algorithm.py2
-rw-r--r--src/sec_certs/model/matching.py2
-rw-r--r--src/sec_certs/sample/cc.py15
-rw-r--r--src/sec_certs/sample/fips.py2
-rw-r--r--src/sec_certs/sample/sar.py2
-rw-r--r--src/sec_certs/utils/extract.py2
-rw-r--r--src/sec_certs/utils/helpers.py6
-rw-r--r--src/sec_certs/utils/pandas.py4
-rw-r--r--src/sec_certs/utils/pdf.py3
-rw-r--r--tests/test_config.py6
21 files changed, 99 insertions, 153 deletions
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
deleted file mode 100644
index 9cb2236f..00000000
--- a/.github/workflows/lint.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-name: Lint (MyPy, Black, Ruff)
-on:
- push:
- workflow_dispatch:
-jobs:
- black:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-python@v4
- with:
- python-version: "3.8"
- - name: Install dependencies
- run: |
- sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
- pip install -r requirements/dev_requirements.txt
- - name: Run Black
- run: black . --check --target-version py38
- ruff:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-python@v4
- with:
- python-version: "3.8"
- - name: Install dependencies
- run: |
- sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
- pip install -r requirements/dev_requirements.txt
- - name: Run Ruff
- run: ruff . --format=github
- mypy:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-python@v4
- with:
- python-version: "3.8"
- - name: Install dependencies
- run: |
- sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
- pip install -r requirements/dev_requirements.txt
- - name: Run Mypy
- run: mypy .
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
new file mode 100644
index 00000000..aa28ddc5
--- /dev/null
+++ b/.github/workflows/pre-commit.yml
@@ -0,0 +1,18 @@
+name: Lint (Ruff, MyPy)
+on:
+ push:
+ workflow_dispatch:
+jobs:
+ pre-commit:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-python@v4
+ with:
+ python-version: "3.8"
+ - name: Install dependencies
+ run: |
+ sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y
+ pip install -r requirements/dev_requirements.txt
+ - name: Run pre-commit
+ run: pre-commit run --all-files
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7725ce34..e9db8430 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,16 +1,12 @@
repos:
- - repo: https://github.com/psf/black
- rev: 23.1.0
- hooks:
- - id: black
- args: ["--check", "--target-version", "py38"]
- - repo: https://github.com/charliermarsh/ruff-pre-commit
- # Ruff version.
- rev: "v0.0.239"
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.1.5
hooks:
- id: ruff
+ - id: ruff-format
+ args: ["--check"]
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: "v1.0.0"
+ rev: "v1.6.1"
hooks:
- id: mypy
additional_dependencies:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ff63e253..148f54b5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -44,9 +44,7 @@ Note on single-sourcing the package version: More can be read [here](https://pac
All commits shall pass the lint pipeline of the following tools:
- Mypy (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings)
-- Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings)
- Ruff (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings)
-- PyUpgrade
These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/main/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool to register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you.
@@ -60,8 +58,8 @@ pre-commit run --all-files
To ivoke the tools manually, you can, in the repository root, use:
- Mypy: `mypy .`
-- Black: `black --check .` (without the flag to reformat)
- Ruff: `ruff ." (or with `--fix` flag to apply fixes)
+- Ruff formatL `ruff format --check .`
## Documentation
diff --git a/notebooks/cc/cert_id_eval.ipynb b/notebooks/cc/cert_id_eval.ipynb
index b558abc2..2af43c54 100644
--- a/notebooks/cc/cert_id_eval.ipynb
+++ b/notebooks/cc/cert_id_eval.ipynb
@@ -39,7 +39,7 @@
"from sec_certs.dataset import CCDataset\n",
"from sec_certs.cert_rules import cc_rules\n",
"import csv\n",
- "import pandas as pd"
+ "import pandas as pd\n"
]
},
{
@@ -51,7 +51,7 @@
"num_schemes = len(cc_rules[\"cc_cert_id\"])\n",
"num_scheme_rules = sum(len(rules) for rules in cc_rules[\"cc_cert_id\"].values())\n",
"print(f\"\\\\newcommand{{\\\\numccschemes}}{{{num_schemes}}}\")\n",
- "print(f\"\\\\newcommand{{\\\\numccschemeidrules}}{{{num_scheme_rules}}}\")"
+ "print(f\"\\\\newcommand{{\\\\numccschemeidrules}}{{{num_scheme_rules}}}\")\n"
]
},
{
@@ -64,7 +64,7 @@
},
"outputs": [],
"source": [
- "dset = CCDataset.from_web_latest()"
+ "dset = CCDataset.from_web_latest()\n"
]
},
{
@@ -75,7 +75,7 @@
"source": [
"num_ids = len(list(filter(lambda cert: cert.heuristics.cert_id, dset)))\n",
"print(f\"\\\\newcommand{{\\\\numcccerts}}{{{len(dset)}}}\")\n",
- "print(f\"\\\\newcommand{{\\\\numccids}}{{{num_ids}}}\")"
+ "print(f\"\\\\newcommand{{\\\\numccids}}{{{num_ids}}}\")\n"
]
},
{
@@ -107,7 +107,7 @@
" if not cert.heuristics.cert_id:\n",
" missing_id_dgsts.add(cert.dgst)\n",
" missing_id.append((cert.dgst, cert.scheme))\n",
- "pd.DataFrame(missing_id, columns=[\"id\", \"scheme\"])"
+ "pd.DataFrame(missing_id, columns=[\"id\", \"scheme\"])\n"
]
},
{
@@ -126,7 +126,7 @@
"missing_manual = pd.read_csv(\"../../data/cert_id_eval/missing_ids.csv\")\n",
"print(set(missing_manual.id) == missing_id_dgsts)\n",
"print(set(missing_manual.id).difference(missing_id_dgsts))\n",
- "print(set(missing_id_dgsts).difference(missing_manual.id))"
+ "print(set(missing_id_dgsts).difference(missing_manual.id))\n"
]
},
{
@@ -144,7 +144,7 @@
"num_missing_manual_unfixable = num_missing_manual - num_missing_manual_fixable\n",
"print(f\"\\\\newcommand{{\\\\numccmissingid}}{{{num_missing_manual}}}\")\n",
"print(f\"\\\\newcommand{{\\\\numccmissingidfixable}}{{{num_missing_manual_fixable}}}\")\n",
- "print(f\"\\\\newcommand{{\\\\numccmissingidunfixable}}{{{num_missing_manual_unfixable}}}\")"
+ "print(f\"\\\\newcommand{{\\\\numccmissingidunfixable}}{{{num_missing_manual_unfixable}}}\")\n"
]
},
{
@@ -153,7 +153,7 @@
"metadata": {},
"outputs": [],
"source": [
- "missing_manual.loc[missing_manual.cert_id.isnull()].reason.value_counts()"
+ "missing_manual.loc[missing_manual.cert_id.isnull()].reason.value_counts()\n"
]
},
{
@@ -162,7 +162,7 @@
"metadata": {},
"outputs": [],
"source": [
- "missing_manual.loc[missing_manual.cert_id.notnull()].reason.value_counts()"
+ "missing_manual.loc[missing_manual.cert_id.notnull()].reason.value_counts()\n"
]
},
{
@@ -195,7 +195,7 @@
"for idd, entries in id_mapping.items():\n",
" if len(entries) > 1 and idd:\n",
" print(idd, entries)\n",
- " duplicate_id_dgsts.update(entries)"
+ " duplicate_id_dgsts.update(entries)\n"
]
},
{
@@ -235,9 +235,9 @@
" print(hash, entries)\n",
" for entry in entries:\n",
" duplicate_doc_dgsts.add(entry)\n",
- " \n",
+ "\n",
"duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n",
- "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)"
+ "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n"
]
},
{
@@ -262,7 +262,7 @@
"outputs": [],
"source": [
"for id in duplicate_ids_issue:\n",
- " print(id, dset[id].heuristics.cert_id)"
+ " print(id, dset[id].heuristics.cert_id)\n"
]
},
{
@@ -273,7 +273,7 @@
"source": [
"print(f\"\\\\newcommand{{\\\\numccduplicateid}}{{{len(duplicate_id_dgsts)}}}\")\n",
"print(f\"\\\\newcommand{{\\\\numccduplicateidcolission}}{{{len(duplicate_ids_due_doc)}}}\")\n",
- "print(f\"\\\\newcommand{{\\\\numccduplicateidissue}}{{{len(duplicate_ids_issue)}}}\")"
+ "print(f\"\\\\newcommand{{\\\\numccduplicateidissue}}{{{len(duplicate_ids_issue)}}}\")\n"
]
},
{
@@ -300,7 +300,7 @@
"duplicate_manual = pd.read_csv(\"../../data/cert_id_eval/duplicate_ids.csv\")\n",
"print(set(duplicate_manual.id) == duplicate_id_dgsts)\n",
"print(set(duplicate_manual.id).difference(duplicate_id_dgsts))\n",
- "print(set(duplicate_id_dgsts).difference(duplicate_manual.id))"
+ "print(set(duplicate_id_dgsts).difference(duplicate_manual.id))\n"
]
},
{
@@ -309,7 +309,7 @@
"metadata": {},
"outputs": [],
"source": [
- "duplicate_manual[duplicate_manual.result == \"tp\"].reason.value_counts()"
+ "duplicate_manual[duplicate_manual.result == \"tp\"].reason.value_counts()\n"
]
},
{
@@ -318,7 +318,7 @@
"metadata": {},
"outputs": [],
"source": [
- "duplicate_manual[duplicate_manual.result == \"fp\"].reason.value_counts()"
+ "duplicate_manual[duplicate_manual.result == \"fp\"].reason.value_counts()\n"
]
},
{
@@ -371,7 +371,7 @@
" impossible.add(line[\"id\"])\n",
" else:\n",
" correct.add(line[\"id\"])\n",
- "print(len(correct), len(possible), len(impossible))"
+ "print(len(correct), len(possible), len(impossible))\n"
]
},
{
@@ -395,7 +395,7 @@
" incorrect.add(line[\"id\"])\n",
" else:\n",
" correct.add(line[\"id\"])\n",
- "print(len(correct), len(incorrect))"
+ "print(len(correct), len(incorrect))\n"
]
},
{
@@ -410,7 +410,7 @@
"source": [
"print(f\"\\\\newcommand{{\\\\numccideval}}{{{len(correct) + len(incorrect)}}}\")\n",
"print(f\"\\\\newcommand{{\\\\numccidevalcorrect}}{{{len(correct)}}}\")\n",
- "print(f\"\\\\newcommand{{\\\\numccidevalincorrect}}{{{len(incorrect)}}}\")"
+ "print(f\"\\\\newcommand{{\\\\numccidevalincorrect}}{{{len(incorrect)}}}\")\n"
]
},
{
@@ -426,7 +426,7 @@
"metadata": {},
"outputs": [],
"source": [
- "manual_references = pd.read_csv(\"../../data/cert_id_eval/random_references.csv\")"
+ "manual_references = pd.read_csv(\"../../data/cert_id_eval/random_references.csv\")\n"
]
},
{
@@ -438,7 +438,7 @@
"print(\"The referenced cert is a...\")\n",
"print(manual_references[manual_references.reason != \"self\"].reason.value_counts())\n",
"print(\"... in the current cert.\")\n",
- "print(\"Total refs:\", sum(manual_references.reason != \"self\"))"
+ "print(\"Total refs:\", sum(manual_references.reason != \"self\"))\n"
]
},
{
@@ -456,7 +456,7 @@
"print(f\"\\\\newcommand{{\\\\numCcRefEvalPrevVersion}}{{{sum(manual_references.reason == 'previous version')}}}\")\n",
"\n",
"print(f\"\\\\newcommand{{\\\\numCcRefEvalInReport}}{{{sum((manual_references.location == 'report') & (manual_references.reason != 'self'))}}}\")\n",
- "print(f\"\\\\newcommand{{\\\\numCcRefEvalInTarget}}{{{sum((manual_references.location == 'target') & (manual_references.reason != 'self'))}}}\")"
+ "print(f\"\\\\newcommand{{\\\\numCcRefEvalInTarget}}{{{sum((manual_references.location == 'target') & (manual_references.reason != 'self'))}}}\")\n"
]
}
],
@@ -476,7 +476,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.3"
+ "version": "3.10.13"
},
"vscode": {
"interpreter": {
diff --git a/pyproject.toml b/pyproject.toml
index 877802eb..e97c5afb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -65,9 +65,8 @@
[project.optional-dependencies]
dev = [
- "black==23.1.0",
- "ruff==0.0.239",
- "mypy==1.0.0",
+ "ruff==0.1.5",
+ "mypy==1.6.1",
"types-PyYAML",
"types-python-dateutil",
"types-requests",
@@ -107,7 +106,7 @@
"SIM",
]
ignore = [
- "E501", # line-length, should be handled by black
+ "E501", # line-length, should be handled by ruff format
]
src = ["src", "tests"]
line-length = 120
@@ -124,25 +123,6 @@
write_to = "src/sec_certs/_version.py"
version_scheme = "no-guess-dev"
-[tool.black]
- line-length = 120
- force-exclude = '''
-/(
- \.git
- | \.mypy_cache
- | \.tox
- | venv
- | \.venv
- | _build
- | buck-out
- | build
- | dist
- | src/sec_certs/_version.py
- | docs
- | .ipynb_checkpoints
- | .eggs
-)/
-'''
[tool.mypy]
plugins = ["numpy.typing.mypy_plugin"]
diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt
index 754a439d..ce4ea166 100644
--- a/requirements/all_requirements.txt
+++ b/requirements/all_requirements.txt
@@ -4,6 +4,10 @@ alabaster==0.7.13
# via sphinx
annotated-types==0.6.0
# via pydantic
+appnope==0.1.3
+ # via
+ # ipykernel
+ # ipython
asttokens==2.4.0
# via stack-data
attrs==23.1.0
@@ -23,8 +27,6 @@ beautifulsoup4==4.12.2
# sec-certs (./../pyproject.toml)
billiard==4.1.0
# via sec-certs (./../pyproject.toml)
-black==23.1.0
- # via sec-certs (./../pyproject.toml)
blis==0.7.11
# via thinc
build==1.0.3
@@ -44,7 +46,6 @@ charset-normalizer==3.3.0
# via requests
click==8.1.7
# via
- # black
# jupyter-cache
# pip-tools
# sec-certs (./../pyproject.toml)
@@ -196,12 +197,10 @@ murmurhash==1.0.10
# preshed
# spacy
# thinc
-mypy==1.0.0
+mypy==1.6.1
# via sec-certs (./../pyproject.toml)
mypy-extensions==1.0.0
- # via
- # black
- # mypy
+ # via mypy
myst-nb==0.17.2
# via sec-certs (./../pyproject.toml)
myst-parser==0.18.1
@@ -237,7 +236,6 @@ numpy==1.26.1
# thinc
packaging==23.2
# via
- # black
# build
# deprecation
# ipykernel
@@ -260,8 +258,6 @@ pandas==2.1.1
# tabula-py
parso==0.8.3
# via jedi
-pathspec==0.11.2
- # via black
pdftotext==2.2.2
# via sec-certs (./../pyproject.toml)
pexpect==4.8.0
@@ -282,7 +278,6 @@ pkgconfig==1.5.5
# via sec-certs (./../pyproject.toml)
platformdirs==3.11.0
# via
- # black
# jupyter-core
# virtualenv
pluggy==1.3.0
@@ -384,7 +379,7 @@ rpds-py==0.10.6
# via
# jsonschema
# referencing
-ruff==0.0.239
+ruff==0.1.5
# via sec-certs (./../pyproject.toml)
scikit-learn==1.3.1
# via sec-certs (./../pyproject.toml)
@@ -470,7 +465,6 @@ threadpoolctl==3.2.0
# via scikit-learn
tomli==2.0.1
# via
- # black
# build
# coverage
# mypy
diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt
index 20164a1a..42ce14d1 100644
--- a/requirements/dev_requirements.txt
+++ b/requirements/dev_requirements.txt
@@ -1,5 +1,9 @@
alabaster==0.7.12
# via sphinx
+appnope==0.1.3
+ # via
+ # ipykernel
+ # ipython
asttokens==2.2.1
# via stack-data
attrs==22.1.0
@@ -17,8 +21,6 @@ beautifulsoup4==4.11.1
# sec-certs (./../pyproject.toml)
billiard==4.0.2
# via sec-certs (./../pyproject.toml)
-black==23.1.0
- # via sec-certs (./../pyproject.toml)
blis==0.7.9
# via thinc
build==0.9.0
@@ -36,7 +38,6 @@ charset-normalizer==2.1.1
# via requests
click==8.1.3
# via
- # black
# jupyter-cache
# pip-tools
# sec-certs (./../pyproject.toml)
@@ -175,12 +176,10 @@ murmurhash==1.0.9
# preshed
# spacy
# thinc
-mypy==1.0.0
+mypy==1.6.1
# via sec-certs (./../pyproject.toml)
-mypy-extensions==0.4.3
- # via
- # black
- # mypy
+mypy-extensions==1.0.0
+ # via mypy
myst-nb==0.17.1
# via sec-certs (./../pyproject.toml)
myst-parser==0.18.1
@@ -219,7 +218,6 @@ numpy==1.23.5
# thinc
packaging==22.0
# via
- # black
# build
# deprecation
# ipykernel
@@ -239,8 +237,6 @@ pandas==1.5.2
# tabula-py
parso==0.8.3
# via jedi
-pathspec==0.10.2
- # via black
pathy==0.10.1
# via spacy
pdftotext==2.2.2
@@ -265,7 +261,6 @@ pkgconfig==1.5.5
# via sec-certs (./../pyproject.toml)
platformdirs==2.6.0
# via
- # black
# jupyter-core
# virtualenv
pluggy==1.0.0
@@ -356,7 +351,7 @@ requests==2.31.0
# sec-certs (./../pyproject.toml)
# spacy
# sphinx
-ruff==0.0.239
+ruff==0.1.5
# via sec-certs (./../pyproject.toml)
scikit-learn==1.2.0
# via sec-certs (./../pyproject.toml)
@@ -436,7 +431,6 @@ toml==0.10.2
# via pre-commit
tomli==2.0.1
# via
- # black
# build
# coverage
# mypy
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index d2664945..cb9d6bba 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -1,3 +1,7 @@
+appnope==0.1.3
+ # via
+ # ipykernel
+ # ipython
asttokens==2.2.1
# via stack-data
attrs==22.1.0
diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt
index 9982c65a..900cbaf0 100644
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@@ -1,3 +1,7 @@
+appnope==0.1.3
+ # via
+ # ipykernel
+ # ipython
asttokens==2.2.1
# via stack-data
attrs==22.1.0
diff --git a/src/sec_certs/dataset/dataset.py b/src/sec_certs/dataset/dataset.py
index 9e49a347..454e86bd 100644
--- a/src/sec_certs/dataset/dataset.py
+++ b/src/sec_certs/dataset/dataset.py
@@ -502,7 +502,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
candidates = [cpe_dset[x].title for x in cert.heuristics.cpe_matches]
candidates += ["No good match"] * (config.cpe_n_max_matches - len(candidates))
options = ["option_" + str(x) for x in range(1, config.cpe_n_max_matches)]
- dct.update({o: c for o, c in zip(options, candidates)})
+ dct.update(dict(zip(options, candidates)))
lst.append(dct)
with Path(output_path).open("w") as handle:
@@ -634,6 +634,6 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl
Enriches the dataset with `certs`
:param List[Certificate] certs: new certs to include into the dataset.
"""
- if any([x not in self for x in certs]):
+ if any(x not in self for x in certs):
logger.warning("Updating dataset with certificates outside of the dataset!")
self.certs.update({x.dgst: x for x in certs})
diff --git a/src/sec_certs/dataset/fips_algorithm.py b/src/sec_certs/dataset/fips_algorithm.py
index fbc8351a..f3f27c0b 100644
--- a/src/sec_certs/dataset/fips_algorithm.py
+++ b/src/sec_certs/dataset/fips_algorithm.py
@@ -82,7 +82,7 @@ class FIPSAlgorithmDataset(JSONPathDataset, ComplexSerializableType):
if failed_tuples:
failed_urls, failed_paths = zip(*failed_tuples)
responses = helpers.download_parallel(failed_urls, failed_paths)
- if any([x != constants.RESPONSE_OK for x in responses]):
+ if any(x != constants.RESPONSE_OK for x in responses):
raise ValueError("Failed to download the algorithms HTML data, the dataset won't be constructed.")
return paths
diff --git a/src/sec_certs/model/matching.py b/src/sec_certs/model/matching.py
index 7e48f878..9c0591f9 100644
--- a/src/sec_certs/model/matching.py
+++ b/src/sec_certs/model/matching.py
@@ -28,7 +28,7 @@ class AbstractMatcher(Generic[CertSubType], ABC):
)
@staticmethod
- def _match_certs(matchers: Sequence["AbstractMatcher"], certs: list[CertSubType], threshold: float):
+ def _match_certs(matchers: Sequence[AbstractMatcher], certs: list[CertSubType], threshold: float):
scores: list[tuple[float, int, int]] = []
matched_is: set[int] = set()
matched_js: set[int] = set()
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index c89f67c6..f2087eeb 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -247,7 +247,7 @@ class CCCertificate(
st_filename: str | None = field(default=None)
def __bool__(self) -> bool:
- return any([x is not None for x in vars(self)])
+ return any(x is not None for x in vars(self))
@property
def bsi_data(self) -> dict[str, Any] | None:
@@ -612,13 +612,12 @@ class CCCertificate(
)
for att, val in vars(self).items():
- if not val:
- setattr(self, att, getattr(other, att))
- elif other_source == "html" and att == "protection_profiles":
- setattr(self, att, getattr(other, att))
- elif other_source == "html" and att == "maintenance_updates":
- setattr(self, att, getattr(other, att))
- elif att == "state":
+ if (
+ (not val)
+ or (other_source == "html" and att == "protection_profiles")
+ or (other_source == "html" and att == "maintenance_updates")
+ or (att == "state")
+ ):
setattr(self, att, getattr(other, att))
else:
if getattr(self, att) != getattr(other, att):
diff --git a/src/sec_certs/sample/fips.py b/src/sec_certs/sample/fips.py
index 26510191..e0bcb74a 100644
--- a/src/sec_certs/sample/fips.py
+++ b/src/sec_certs/sample/fips.py
@@ -64,7 +64,7 @@ class FIPSHTMLParser:
)
entries = [(FIPSHTMLParser.normalize_string(key.text), entry) for key, entry in entries]
entries = [parse_single_detail_entry(*x) for x in entries if x[0] in DETAILS_KEY_NORMALIZATION_DICT]
- entries = {x: y for x, y in entries}
+ entries = dict(entries)
if "caveat" in entries:
entries["mentioned_certs"] = FIPSHTMLParser.get_mentioned_certs_from_caveat(entries["caveat"])
diff --git a/src/sec_certs/sample/sar.py b/src/sec_certs/sample/sar.py
index 8f48f417..e3d7bd2b 100644
--- a/src/sec_certs/sample/sar.py
+++ b/src/sec_certs/sample/sar.py
@@ -50,7 +50,7 @@ class SAR(ComplexSerializableType):
@staticmethod
def matches_re(string: str) -> bool:
return any(
- [re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING]
+ re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING
)
def __lt__(self, other: Any) -> bool:
diff --git a/src/sec_certs/utils/extract.py b/src/sec_certs/utils/extract.py
index 933db4b8..e6312b8d 100644
--- a/src/sec_certs/utils/extract.py
+++ b/src/sec_certs/utils/extract.py
@@ -353,7 +353,7 @@ def search_only_headers_bsi(filepath: Path): # noqa: C901
for m in re.finditer(rule_and_sep, whole_text):
# check if previous rules had at least one match
- if constants.TAG_CERT_ID not in items_found.keys():
+ if constants.TAG_CERT_ID not in items_found:
logger.error(f"ERROR: front page not found for file: {filepath}")
match_groups = m.groups()
diff --git a/src/sec_certs/utils/helpers.py b/src/sec_certs/utils/helpers.py
index 596ecf62..c103a2d3 100644
--- a/src/sec_certs/utils/helpers.py
+++ b/src/sec_certs/utils/helpers.py
@@ -28,7 +28,11 @@ def download_file(
time.sleep(delay)
# See https://github.com/psf/requests/issues/3953 for header justification
r = requests.get(
- url, allow_redirects=True, timeout=constants.REQUEST_TIMEOUT, stream=True, headers={"Accept-Encoding": None} # type: ignore
+ url,
+ allow_redirects=True,
+ timeout=constants.REQUEST_TIMEOUT,
+ stream=True,
+ headers={"Accept-Encoding": None}, # type: ignore
)
ctx: Any
if show_progress_bar:
diff --git a/src/sec_certs/utils/pandas.py b/src/sec_certs/utils/pandas.py
index 5b0a668d..661b669c 100644
--- a/src/sec_certs/utils/pandas.py
+++ b/src/sec_certs/utils/pandas.py
@@ -141,7 +141,7 @@ def get_sar_level_from_set(sars: set[SAR], sar_family: str) -> int | None:
Given a set of SARs and a family name, will return level of the seeked SAR from the set.
"""
family_sars_dict = {x.family: x for x in sars} if (sars and not pd.isnull(sars)) else {}
- if sar_family not in family_sars_dict.keys():
+ if sar_family not in family_sars_dict:
return None
return family_sars_dict[sar_family].level
@@ -211,7 +211,7 @@ def compute_cve_correlations(
tuples = list(
zip(n_cves_corrs, n_cves_pvalues, worst_cve_corrs, worst_cve_pvalues, avg_cve_corrs, avg_cve_pvalues, supports)
)
- dct = {family: correlations for family, correlations in zip(["eal"] + families, tuples)}
+ dct = dict(zip(["eal"] + families, tuples))
df_corr = pd.DataFrame.from_dict(
dct,
orient="index",
diff --git a/src/sec_certs/utils/pdf.py b/src/sec_certs/utils/pdf.py
index 7736b5d0..f2c2c58e 100644
--- a/src/sec_certs/utils/pdf.py
+++ b/src/sec_certs/utils/pdf.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import glob
import logging
import subprocess
from datetime import datetime, timedelta, timezone
@@ -54,7 +53,7 @@ def ocr_pdf_file(pdf_path: Path) -> str:
if ppm.returncode != 0:
raise ValueError(f"pdftoppm failed: {ppm.returncode}")
- for ppm_path in map(Path, glob.glob(str(tmppath / "image*.ppm"))):
+ for ppm_path in tmppath.rglob("image*.ppm"):
base = ppm_path.with_suffix("")
content = pytesseract.image_to_string(Image.open(ppm_path), lang="eng+deu+fra")
diff --git a/tests/test_config.py b/tests/test_config.py
index a9099327..54e5a8d5 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -42,8 +42,8 @@ def test_config_from_yaml(simple_config_dict, simple_config_yaml: Path) -> None:
def test_load_env_values(simple_config_dict, simple_config_yaml):
- os.environ["seccerts_log_filepath"] = "/some/nonsense/path"
- os.environ["always_false_positive_fips_cert_id_threshold"] = "10"
+ os.environ["SECCERTS_LOG_FILEPATH"] = "/some/nonsense/path"
+ os.environ["ALWAYS_FALSE_POSITIVE_FIPS_CERT_ID_THRESHOLD"] = "10"
config_module.config.load_from_yaml(simple_config_yaml)
@@ -57,7 +57,7 @@ def test_load_env_values(simple_config_dict, simple_config_yaml):
def test_complex_config_load(simple_config_dict, simple_config_yaml):
config_module.config.year_difference_between_validations = 123456789
config_module.config.n_threads = 987654321
- os.environ["seccerts_n_threads"] = "1"
+ os.environ["SECCERTS_N_THREADS"] = "1"
config_module.config.load_from_yaml(simple_config_yaml)
for key, val in simple_config_dict.items():