diff options
| author | adamjanovsky | 2023-11-10 12:54:00 +0100 |
|---|---|---|
| committer | GitHub | 2023-11-10 12:54:00 +0100 |
| commit | 1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e (patch) | |
| tree | c0c2f35f921f3551f2bf0a146c1cbd6c0494f7a9 | |
| parent | 5968e53d9f78d744c1bd151be0d754282e65baee (diff) | |
| parent | 10aa312c5e83c197a2d41fec710c5301bfff169e (diff) | |
| download | sec-certs-1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e.tar.gz sec-certs-1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e.tar.zst sec-certs-1ccca9ae8afa8e6574e1cbba2c93b8d5428e2b2e.zip | |
Merge pull request #373 from crocs-muni/switch-black-to-ruff
switch black -> ruff format
| -rw-r--r-- | .github/workflows/lint.yml | 44 | ||||
| -rw-r--r-- | .github/workflows/pre-commit.yml | 18 | ||||
| -rw-r--r-- | .pre-commit-config.yaml | 14 | ||||
| -rw-r--r-- | CONTRIBUTING.md | 4 | ||||
| -rw-r--r-- | notebooks/cc/cert_id_eval.ipynb | 48 | ||||
| -rw-r--r-- | pyproject.toml | 26 | ||||
| -rw-r--r-- | requirements/all_requirements.txt | 20 | ||||
| -rw-r--r-- | requirements/dev_requirements.txt | 22 | ||||
| -rw-r--r-- | requirements/requirements.txt | 4 | ||||
| -rw-r--r-- | requirements/test_requirements.txt | 4 | ||||
| -rw-r--r-- | src/sec_certs/dataset/dataset.py | 4 | ||||
| -rw-r--r-- | src/sec_certs/dataset/fips_algorithm.py | 2 | ||||
| -rw-r--r-- | src/sec_certs/model/matching.py | 2 | ||||
| -rw-r--r-- | src/sec_certs/sample/cc.py | 15 | ||||
| -rw-r--r-- | src/sec_certs/sample/fips.py | 2 | ||||
| -rw-r--r-- | src/sec_certs/sample/sar.py | 2 | ||||
| -rw-r--r-- | src/sec_certs/utils/extract.py | 2 | ||||
| -rw-r--r-- | src/sec_certs/utils/helpers.py | 6 | ||||
| -rw-r--r-- | src/sec_certs/utils/pandas.py | 4 | ||||
| -rw-r--r-- | src/sec_certs/utils/pdf.py | 3 | ||||
| -rw-r--r-- | tests/test_config.py | 6 |
21 files changed, 99 insertions, 153 deletions
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 9cb2236f..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Lint (MyPy, Black, Ruff) -on: - push: - workflow_dispatch: -jobs: - black: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Install dependencies - run: | - sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y - pip install -r requirements/dev_requirements.txt - - name: Run Black - run: black . --check --target-version py38 - ruff: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Install dependencies - run: | - sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y - pip install -r requirements/dev_requirements.txt - - name: Run Ruff - run: ruff . --format=github - mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Install dependencies - run: | - sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y - pip install -r requirements/dev_requirements.txt - - name: Run Mypy - run: mypy . diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000..aa28ddc5 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,18 @@ +name: Lint (Ruff, MyPy) +on: + push: + workflow_dispatch: +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install dependencies + run: | + sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev -y + pip install -r requirements/dev_requirements.txt + - name: Run pre-commit + run: pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7725ce34..e9db8430 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,12 @@ repos: - - repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - args: ["--check", "--target-version", "py38"] - - repo: https://github.com/charliermarsh/ruff-pre-commit - # Ruff version. - rev: "v0.0.239" + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.5 hooks: - id: ruff + - id: ruff-format + args: ["--check"] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.0.0" + rev: "v1.6.1" hooks: - id: mypy additional_dependencies: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff63e253..148f54b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,9 +44,7 @@ Note on single-sourcing the package version: More can be read [here](https://pac All commits shall pass the lint pipeline of the following tools: - Mypy (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) -- Black (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) - Ruff (see [pyproject.toml](https://github.com/crocs-muni/sec-certs/blob/main/pyproject.toml) for settings) -- PyUpgrade These tools can be installed via [dev_requirements.txt](https://github.com/crocs-muni/sec-certs/blob/main/dev_requirements.txt) You can use [pre-commit](https://pre-commit.com/) tool to register git hook that will evalute these checks prior to any commit and abort the commit for you. Note that the pre-commit is not meant to automatically fix the issues, just warn you. @@ -60,8 +58,8 @@ pre-commit run --all-files To ivoke the tools manually, you can, in the repository root, use: - Mypy: `mypy .` -- Black: `black --check .` (without the flag to reformat) - Ruff: `ruff ." (or with `--fix` flag to apply fixes) +- Ruff formatL `ruff format --check .` ## Documentation diff --git a/notebooks/cc/cert_id_eval.ipynb b/notebooks/cc/cert_id_eval.ipynb index b558abc2..2af43c54 100644 --- a/notebooks/cc/cert_id_eval.ipynb +++ b/notebooks/cc/cert_id_eval.ipynb @@ -39,7 +39,7 @@ "from sec_certs.dataset import CCDataset\n", "from sec_certs.cert_rules import cc_rules\n", "import csv\n", - "import pandas as pd" + "import pandas as pd\n" ] }, { @@ -51,7 +51,7 @@ "num_schemes = len(cc_rules[\"cc_cert_id\"])\n", "num_scheme_rules = sum(len(rules) for rules in cc_rules[\"cc_cert_id\"].values())\n", "print(f\"\\\\newcommand{{\\\\numccschemes}}{{{num_schemes}}}\")\n", - "print(f\"\\\\newcommand{{\\\\numccschemeidrules}}{{{num_scheme_rules}}}\")" + "print(f\"\\\\newcommand{{\\\\numccschemeidrules}}{{{num_scheme_rules}}}\")\n" ] }, { @@ -64,7 +64,7 @@ }, "outputs": [], "source": [ - "dset = CCDataset.from_web_latest()" + "dset = CCDataset.from_web_latest()\n" ] }, { @@ -75,7 +75,7 @@ "source": [ "num_ids = len(list(filter(lambda cert: cert.heuristics.cert_id, dset)))\n", "print(f\"\\\\newcommand{{\\\\numcccerts}}{{{len(dset)}}}\")\n", - "print(f\"\\\\newcommand{{\\\\numccids}}{{{num_ids}}}\")" + "print(f\"\\\\newcommand{{\\\\numccids}}{{{num_ids}}}\")\n" ] }, { @@ -107,7 +107,7 @@ " if not cert.heuristics.cert_id:\n", " missing_id_dgsts.add(cert.dgst)\n", " missing_id.append((cert.dgst, cert.scheme))\n", - "pd.DataFrame(missing_id, columns=[\"id\", \"scheme\"])" + "pd.DataFrame(missing_id, columns=[\"id\", \"scheme\"])\n" ] }, { @@ -126,7 +126,7 @@ "missing_manual = pd.read_csv(\"../../data/cert_id_eval/missing_ids.csv\")\n", "print(set(missing_manual.id) == missing_id_dgsts)\n", "print(set(missing_manual.id).difference(missing_id_dgsts))\n", - "print(set(missing_id_dgsts).difference(missing_manual.id))" + "print(set(missing_id_dgsts).difference(missing_manual.id))\n" ] }, { @@ -144,7 +144,7 @@ "num_missing_manual_unfixable = num_missing_manual - num_missing_manual_fixable\n", "print(f\"\\\\newcommand{{\\\\numccmissingid}}{{{num_missing_manual}}}\")\n", "print(f\"\\\\newcommand{{\\\\numccmissingidfixable}}{{{num_missing_manual_fixable}}}\")\n", - "print(f\"\\\\newcommand{{\\\\numccmissingidunfixable}}{{{num_missing_manual_unfixable}}}\")" + "print(f\"\\\\newcommand{{\\\\numccmissingidunfixable}}{{{num_missing_manual_unfixable}}}\")\n" ] }, { @@ -153,7 +153,7 @@ "metadata": {}, "outputs": [], "source": [ - "missing_manual.loc[missing_manual.cert_id.isnull()].reason.value_counts()" + "missing_manual.loc[missing_manual.cert_id.isnull()].reason.value_counts()\n" ] }, { @@ -162,7 +162,7 @@ "metadata": {}, "outputs": [], "source": [ - "missing_manual.loc[missing_manual.cert_id.notnull()].reason.value_counts()" + "missing_manual.loc[missing_manual.cert_id.notnull()].reason.value_counts()\n" ] }, { @@ -195,7 +195,7 @@ "for idd, entries in id_mapping.items():\n", " if len(entries) > 1 and idd:\n", " print(idd, entries)\n", - " duplicate_id_dgsts.update(entries)" + " duplicate_id_dgsts.update(entries)\n" ] }, { @@ -235,9 +235,9 @@ " print(hash, entries)\n", " for entry in entries:\n", " duplicate_doc_dgsts.add(entry)\n", - " \n", + "\n", "duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n", - "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)" + "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n" ] }, { @@ -262,7 +262,7 @@ "outputs": [], "source": [ "for id in duplicate_ids_issue:\n", - " print(id, dset[id].heuristics.cert_id)" + " print(id, dset[id].heuristics.cert_id)\n" ] }, { @@ -273,7 +273,7 @@ "source": [ "print(f\"\\\\newcommand{{\\\\numccduplicateid}}{{{len(duplicate_id_dgsts)}}}\")\n", "print(f\"\\\\newcommand{{\\\\numccduplicateidcolission}}{{{len(duplicate_ids_due_doc)}}}\")\n", - "print(f\"\\\\newcommand{{\\\\numccduplicateidissue}}{{{len(duplicate_ids_issue)}}}\")" + "print(f\"\\\\newcommand{{\\\\numccduplicateidissue}}{{{len(duplicate_ids_issue)}}}\")\n" ] }, { @@ -300,7 +300,7 @@ "duplicate_manual = pd.read_csv(\"../../data/cert_id_eval/duplicate_ids.csv\")\n", "print(set(duplicate_manual.id) == duplicate_id_dgsts)\n", "print(set(duplicate_manual.id).difference(duplicate_id_dgsts))\n", - "print(set(duplicate_id_dgsts).difference(duplicate_manual.id))" + "print(set(duplicate_id_dgsts).difference(duplicate_manual.id))\n" ] }, { @@ -309,7 +309,7 @@ "metadata": {}, "outputs": [], "source": [ - "duplicate_manual[duplicate_manual.result == \"tp\"].reason.value_counts()" + "duplicate_manual[duplicate_manual.result == \"tp\"].reason.value_counts()\n" ] }, { @@ -318,7 +318,7 @@ "metadata": {}, "outputs": [], "source": [ - "duplicate_manual[duplicate_manual.result == \"fp\"].reason.value_counts()" + "duplicate_manual[duplicate_manual.result == \"fp\"].reason.value_counts()\n" ] }, { @@ -371,7 +371,7 @@ " impossible.add(line[\"id\"])\n", " else:\n", " correct.add(line[\"id\"])\n", - "print(len(correct), len(possible), len(impossible))" + "print(len(correct), len(possible), len(impossible))\n" ] }, { @@ -395,7 +395,7 @@ " incorrect.add(line[\"id\"])\n", " else:\n", " correct.add(line[\"id\"])\n", - "print(len(correct), len(incorrect))" + "print(len(correct), len(incorrect))\n" ] }, { @@ -410,7 +410,7 @@ "source": [ "print(f\"\\\\newcommand{{\\\\numccideval}}{{{len(correct) + len(incorrect)}}}\")\n", "print(f\"\\\\newcommand{{\\\\numccidevalcorrect}}{{{len(correct)}}}\")\n", - "print(f\"\\\\newcommand{{\\\\numccidevalincorrect}}{{{len(incorrect)}}}\")" + "print(f\"\\\\newcommand{{\\\\numccidevalincorrect}}{{{len(incorrect)}}}\")\n" ] }, { @@ -426,7 +426,7 @@ "metadata": {}, "outputs": [], "source": [ - "manual_references = pd.read_csv(\"../../data/cert_id_eval/random_references.csv\")" + "manual_references = pd.read_csv(\"../../data/cert_id_eval/random_references.csv\")\n" ] }, { @@ -438,7 +438,7 @@ "print(\"The referenced cert is a...\")\n", "print(manual_references[manual_references.reason != \"self\"].reason.value_counts())\n", "print(\"... in the current cert.\")\n", - "print(\"Total refs:\", sum(manual_references.reason != \"self\"))" + "print(\"Total refs:\", sum(manual_references.reason != \"self\"))\n" ] }, { @@ -456,7 +456,7 @@ "print(f\"\\\\newcommand{{\\\\numCcRefEvalPrevVersion}}{{{sum(manual_references.reason == 'previous version')}}}\")\n", "\n", "print(f\"\\\\newcommand{{\\\\numCcRefEvalInReport}}{{{sum((manual_references.location == 'report') & (manual_references.reason != 'self'))}}}\")\n", - "print(f\"\\\\newcommand{{\\\\numCcRefEvalInTarget}}{{{sum((manual_references.location == 'target') & (manual_references.reason != 'self'))}}}\")" + "print(f\"\\\\newcommand{{\\\\numCcRefEvalInTarget}}{{{sum((manual_references.location == 'target') & (manual_references.reason != 'self'))}}}\")\n" ] } ], @@ -476,7 +476,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.10.13" }, "vscode": { "interpreter": { diff --git a/pyproject.toml b/pyproject.toml index 877802eb..e97c5afb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,9 +65,8 @@ [project.optional-dependencies] dev = [ - "black==23.1.0", - "ruff==0.0.239", - "mypy==1.0.0", + "ruff==0.1.5", + "mypy==1.6.1", "types-PyYAML", "types-python-dateutil", "types-requests", @@ -107,7 +106,7 @@ "SIM", ] ignore = [ - "E501", # line-length, should be handled by black + "E501", # line-length, should be handled by ruff format ] src = ["src", "tests"] line-length = 120 @@ -124,25 +123,6 @@ write_to = "src/sec_certs/_version.py" version_scheme = "no-guess-dev" -[tool.black] - line-length = 120 - force-exclude = ''' -/( - \.git - | \.mypy_cache - | \.tox - | venv - | \.venv - | _build - | buck-out - | build - | dist - | src/sec_certs/_version.py - | docs - | .ipynb_checkpoints - | .eggs -)/ -''' [tool.mypy] plugins = ["numpy.typing.mypy_plugin"] diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt index 754a439d..ce4ea166 100644 --- a/requirements/all_requirements.txt +++ b/requirements/all_requirements.txt @@ -4,6 +4,10 @@ alabaster==0.7.13 # via sphinx annotated-types==0.6.0 # via pydantic +appnope==0.1.3 + # via + # ipykernel + # ipython asttokens==2.4.0 # via stack-data attrs==23.1.0 @@ -23,8 +27,6 @@ beautifulsoup4==4.12.2 # sec-certs (./../pyproject.toml) billiard==4.1.0 # via sec-certs (./../pyproject.toml) -black==23.1.0 - # via sec-certs (./../pyproject.toml) blis==0.7.11 # via thinc build==1.0.3 @@ -44,7 +46,6 @@ charset-normalizer==3.3.0 # via requests click==8.1.7 # via - # black # jupyter-cache # pip-tools # sec-certs (./../pyproject.toml) @@ -196,12 +197,10 @@ murmurhash==1.0.10 # preshed # spacy # thinc -mypy==1.0.0 +mypy==1.6.1 # via sec-certs (./../pyproject.toml) mypy-extensions==1.0.0 - # via - # black - # mypy + # via mypy myst-nb==0.17.2 # via sec-certs (./../pyproject.toml) myst-parser==0.18.1 @@ -237,7 +236,6 @@ numpy==1.26.1 # thinc packaging==23.2 # via - # black # build # deprecation # ipykernel @@ -260,8 +258,6 @@ pandas==2.1.1 # tabula-py parso==0.8.3 # via jedi -pathspec==0.11.2 - # via black pdftotext==2.2.2 # via sec-certs (./../pyproject.toml) pexpect==4.8.0 @@ -282,7 +278,6 @@ pkgconfig==1.5.5 # via sec-certs (./../pyproject.toml) platformdirs==3.11.0 # via - # black # jupyter-core # virtualenv pluggy==1.3.0 @@ -384,7 +379,7 @@ rpds-py==0.10.6 # via # jsonschema # referencing -ruff==0.0.239 +ruff==0.1.5 # via sec-certs (./../pyproject.toml) scikit-learn==1.3.1 # via sec-certs (./../pyproject.toml) @@ -470,7 +465,6 @@ threadpoolctl==3.2.0 # via scikit-learn tomli==2.0.1 # via - # black # build # coverage # mypy diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index 20164a1a..42ce14d1 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -1,5 +1,9 @@ alabaster==0.7.12 # via sphinx +appnope==0.1.3 + # via + # ipykernel + # ipython asttokens==2.2.1 # via stack-data attrs==22.1.0 @@ -17,8 +21,6 @@ beautifulsoup4==4.11.1 # sec-certs (./../pyproject.toml) billiard==4.0.2 # via sec-certs (./../pyproject.toml) -black==23.1.0 - # via sec-certs (./../pyproject.toml) blis==0.7.9 # via thinc build==0.9.0 @@ -36,7 +38,6 @@ charset-normalizer==2.1.1 # via requests click==8.1.3 # via - # black # jupyter-cache # pip-tools # sec-certs (./../pyproject.toml) @@ -175,12 +176,10 @@ murmurhash==1.0.9 # preshed # spacy # thinc -mypy==1.0.0 +mypy==1.6.1 # via sec-certs (./../pyproject.toml) -mypy-extensions==0.4.3 - # via - # black - # mypy +mypy-extensions==1.0.0 + # via mypy myst-nb==0.17.1 # via sec-certs (./../pyproject.toml) myst-parser==0.18.1 @@ -219,7 +218,6 @@ numpy==1.23.5 # thinc packaging==22.0 # via - # black # build # deprecation # ipykernel @@ -239,8 +237,6 @@ pandas==1.5.2 # tabula-py parso==0.8.3 # via jedi -pathspec==0.10.2 - # via black pathy==0.10.1 # via spacy pdftotext==2.2.2 @@ -265,7 +261,6 @@ pkgconfig==1.5.5 # via sec-certs (./../pyproject.toml) platformdirs==2.6.0 # via - # black # jupyter-core # virtualenv pluggy==1.0.0 @@ -356,7 +351,7 @@ requests==2.31.0 # sec-certs (./../pyproject.toml) # spacy # sphinx -ruff==0.0.239 +ruff==0.1.5 # via sec-certs (./../pyproject.toml) scikit-learn==1.2.0 # via sec-certs (./../pyproject.toml) @@ -436,7 +431,6 @@ toml==0.10.2 # via pre-commit tomli==2.0.1 # via - # black # build # coverage # mypy diff --git a/requirements/requirements.txt b/requirements/requirements.txt index d2664945..cb9d6bba 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,3 +1,7 @@ +appnope==0.1.3 + # via + # ipykernel + # ipython asttokens==2.2.1 # via stack-data attrs==22.1.0 diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index 9982c65a..900cbaf0 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -1,3 +1,7 @@ +appnope==0.1.3 + # via + # ipykernel + # ipython asttokens==2.2.1 # via stack-data attrs==22.1.0 diff --git a/src/sec_certs/dataset/dataset.py b/src/sec_certs/dataset/dataset.py index 9e49a347..454e86bd 100644 --- a/src/sec_certs/dataset/dataset.py +++ b/src/sec_certs/dataset/dataset.py @@ -502,7 +502,7 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl candidates = [cpe_dset[x].title for x in cert.heuristics.cpe_matches] candidates += ["No good match"] * (config.cpe_n_max_matches - len(candidates)) options = ["option_" + str(x) for x in range(1, config.cpe_n_max_matches)] - dct.update({o: c for o, c in zip(options, candidates)}) + dct.update(dict(zip(options, candidates))) lst.append(dct) with Path(output_path).open("w") as handle: @@ -634,6 +634,6 @@ class Dataset(Generic[CertSubType, AuxiliaryDatasetsSubType], ComplexSerializabl Enriches the dataset with `certs` :param List[Certificate] certs: new certs to include into the dataset. """ - if any([x not in self for x in certs]): + if any(x not in self for x in certs): logger.warning("Updating dataset with certificates outside of the dataset!") self.certs.update({x.dgst: x for x in certs}) diff --git a/src/sec_certs/dataset/fips_algorithm.py b/src/sec_certs/dataset/fips_algorithm.py index fbc8351a..f3f27c0b 100644 --- a/src/sec_certs/dataset/fips_algorithm.py +++ b/src/sec_certs/dataset/fips_algorithm.py @@ -82,7 +82,7 @@ class FIPSAlgorithmDataset(JSONPathDataset, ComplexSerializableType): if failed_tuples: failed_urls, failed_paths = zip(*failed_tuples) responses = helpers.download_parallel(failed_urls, failed_paths) - if any([x != constants.RESPONSE_OK for x in responses]): + if any(x != constants.RESPONSE_OK for x in responses): raise ValueError("Failed to download the algorithms HTML data, the dataset won't be constructed.") return paths diff --git a/src/sec_certs/model/matching.py b/src/sec_certs/model/matching.py index 7e48f878..9c0591f9 100644 --- a/src/sec_certs/model/matching.py +++ b/src/sec_certs/model/matching.py @@ -28,7 +28,7 @@ class AbstractMatcher(Generic[CertSubType], ABC): ) @staticmethod - def _match_certs(matchers: Sequence["AbstractMatcher"], certs: list[CertSubType], threshold: float): + def _match_certs(matchers: Sequence[AbstractMatcher], certs: list[CertSubType], threshold: float): scores: list[tuple[float, int, int]] = [] matched_is: set[int] = set() matched_js: set[int] = set() diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py index c89f67c6..f2087eeb 100644 --- a/src/sec_certs/sample/cc.py +++ b/src/sec_certs/sample/cc.py @@ -247,7 +247,7 @@ class CCCertificate( st_filename: str | None = field(default=None) def __bool__(self) -> bool: - return any([x is not None for x in vars(self)]) + return any(x is not None for x in vars(self)) @property def bsi_data(self) -> dict[str, Any] | None: @@ -612,13 +612,12 @@ class CCCertificate( ) for att, val in vars(self).items(): - if not val: - setattr(self, att, getattr(other, att)) - elif other_source == "html" and att == "protection_profiles": - setattr(self, att, getattr(other, att)) - elif other_source == "html" and att == "maintenance_updates": - setattr(self, att, getattr(other, att)) - elif att == "state": + if ( + (not val) + or (other_source == "html" and att == "protection_profiles") + or (other_source == "html" and att == "maintenance_updates") + or (att == "state") + ): setattr(self, att, getattr(other, att)) else: if getattr(self, att) != getattr(other, att): diff --git a/src/sec_certs/sample/fips.py b/src/sec_certs/sample/fips.py index 26510191..e0bcb74a 100644 --- a/src/sec_certs/sample/fips.py +++ b/src/sec_certs/sample/fips.py @@ -64,7 +64,7 @@ class FIPSHTMLParser: ) entries = [(FIPSHTMLParser.normalize_string(key.text), entry) for key, entry in entries] entries = [parse_single_detail_entry(*x) for x in entries if x[0] in DETAILS_KEY_NORMALIZATION_DICT] - entries = {x: y for x, y in entries} + entries = dict(entries) if "caveat" in entries: entries["mentioned_certs"] = FIPSHTMLParser.get_mentioned_certs_from_caveat(entries["caveat"]) diff --git a/src/sec_certs/sample/sar.py b/src/sec_certs/sample/sar.py index 8f48f417..e3d7bd2b 100644 --- a/src/sec_certs/sample/sar.py +++ b/src/sec_certs/sample/sar.py @@ -50,7 +50,7 @@ class SAR(ComplexSerializableType): @staticmethod def matches_re(string: str) -> bool: return any( - [re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING] + re.match(sar_class + "(?:_[A-Z]{3,4}){1,2}(?:\\.[0-9]){0,2}", string) for sar_class in SAR_CLASS_MAPPING ) def __lt__(self, other: Any) -> bool: diff --git a/src/sec_certs/utils/extract.py b/src/sec_certs/utils/extract.py index 933db4b8..e6312b8d 100644 --- a/src/sec_certs/utils/extract.py +++ b/src/sec_certs/utils/extract.py @@ -353,7 +353,7 @@ def search_only_headers_bsi(filepath: Path): # noqa: C901 for m in re.finditer(rule_and_sep, whole_text): # check if previous rules had at least one match - if constants.TAG_CERT_ID not in items_found.keys(): + if constants.TAG_CERT_ID not in items_found: logger.error(f"ERROR: front page not found for file: {filepath}") match_groups = m.groups() diff --git a/src/sec_certs/utils/helpers.py b/src/sec_certs/utils/helpers.py index 596ecf62..c103a2d3 100644 --- a/src/sec_certs/utils/helpers.py +++ b/src/sec_certs/utils/helpers.py @@ -28,7 +28,11 @@ def download_file( time.sleep(delay) # See https://github.com/psf/requests/issues/3953 for header justification r = requests.get( - url, allow_redirects=True, timeout=constants.REQUEST_TIMEOUT, stream=True, headers={"Accept-Encoding": None} # type: ignore + url, + allow_redirects=True, + timeout=constants.REQUEST_TIMEOUT, + stream=True, + headers={"Accept-Encoding": None}, # type: ignore ) ctx: Any if show_progress_bar: diff --git a/src/sec_certs/utils/pandas.py b/src/sec_certs/utils/pandas.py index 5b0a668d..661b669c 100644 --- a/src/sec_certs/utils/pandas.py +++ b/src/sec_certs/utils/pandas.py @@ -141,7 +141,7 @@ def get_sar_level_from_set(sars: set[SAR], sar_family: str) -> int | None: Given a set of SARs and a family name, will return level of the seeked SAR from the set. """ family_sars_dict = {x.family: x for x in sars} if (sars and not pd.isnull(sars)) else {} - if sar_family not in family_sars_dict.keys(): + if sar_family not in family_sars_dict: return None return family_sars_dict[sar_family].level @@ -211,7 +211,7 @@ def compute_cve_correlations( tuples = list( zip(n_cves_corrs, n_cves_pvalues, worst_cve_corrs, worst_cve_pvalues, avg_cve_corrs, avg_cve_pvalues, supports) ) - dct = {family: correlations for family, correlations in zip(["eal"] + families, tuples)} + dct = dict(zip(["eal"] + families, tuples)) df_corr = pd.DataFrame.from_dict( dct, orient="index", diff --git a/src/sec_certs/utils/pdf.py b/src/sec_certs/utils/pdf.py index 7736b5d0..f2c2c58e 100644 --- a/src/sec_certs/utils/pdf.py +++ b/src/sec_certs/utils/pdf.py @@ -1,6 +1,5 @@ from __future__ import annotations -import glob import logging import subprocess from datetime import datetime, timedelta, timezone @@ -54,7 +53,7 @@ def ocr_pdf_file(pdf_path: Path) -> str: if ppm.returncode != 0: raise ValueError(f"pdftoppm failed: {ppm.returncode}") - for ppm_path in map(Path, glob.glob(str(tmppath / "image*.ppm"))): + for ppm_path in tmppath.rglob("image*.ppm"): base = ppm_path.with_suffix("") content = pytesseract.image_to_string(Image.open(ppm_path), lang="eng+deu+fra") diff --git a/tests/test_config.py b/tests/test_config.py index a9099327..54e5a8d5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -42,8 +42,8 @@ def test_config_from_yaml(simple_config_dict, simple_config_yaml: Path) -> None: def test_load_env_values(simple_config_dict, simple_config_yaml): - os.environ["seccerts_log_filepath"] = "/some/nonsense/path" - os.environ["always_false_positive_fips_cert_id_threshold"] = "10" + os.environ["SECCERTS_LOG_FILEPATH"] = "/some/nonsense/path" + os.environ["ALWAYS_FALSE_POSITIVE_FIPS_CERT_ID_THRESHOLD"] = "10" config_module.config.load_from_yaml(simple_config_yaml) @@ -57,7 +57,7 @@ def test_load_env_values(simple_config_dict, simple_config_yaml): def test_complex_config_load(simple_config_dict, simple_config_yaml): config_module.config.year_difference_between_validations = 123456789 config_module.config.n_threads = 987654321 - os.environ["seccerts_n_threads"] = "1" + os.environ["SECCERTS_N_THREADS"] = "1" config_module.config.load_from_yaml(simple_config_yaml) for key, val in simple_config_dict.items(): |
