From 450b7dd65f095b98704a379ef7b654890098b33c Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 18 Dec 2021 14:02:25 +0100 Subject: Format code with black --- tests/test_cc_txt_processing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/test_cc_txt_processing.py') diff --git a/tests/test_cc_txt_processing.py b/tests/test_cc_txt_processing.py index 01b2212b..96716fbe 100644 --- a/tests/test_cc_txt_processing.py +++ b/tests/test_cc_txt_processing.py @@ -10,7 +10,7 @@ from sec_certs.sample.common_criteria import CommonCriteriaCert class TestCommonCriteriaHeuristics(TestCase): - dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / 'vulnerable_dataset.json' + dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / "vulnerable_dataset.json" data_dir_path: ClassVar[Path] = dataset_json_path.parent @classmethod @@ -18,7 +18,7 @@ class TestCommonCriteriaHeuristics(TestCase): cls.tmp_dir: ClassVar[tempfile.TemporaryDirectory] = tempfile.TemporaryDirectory() shutil.copytree(cls.data_dir_path, cls.tmp_dir.name, dirs_exist_ok=True) - cls.cc_dset: CCDataset = CCDataset.from_json(Path(cls.tmp_dir.name) / 'vulnerable_dataset.json') + cls.cc_dset: CCDataset = CCDataset.from_json(Path(cls.tmp_dir.name) / "vulnerable_dataset.json") cls.cc_dset.download_all_pdfs() cls.cc_dset.convert_all_pdfs() -- cgit v1.3.1 From 6e980798f0e51ac2b581aa79c8d314a3c935b4c5 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 18 Dec 2021 14:18:46 +0100 Subject: sort imports --- cc_cli.py | 7 ++--- examples/cc_cpe_labeling.py | 6 ++--- examples/cc_oop_demo.py | 6 ++--- examples/fips_cpe_labeling.py | 7 ++--- examples/fips_oop_demo.py | 9 ++++--- fips_cli.py | 11 ++++---- pyproject.toml | 3 ++- sec_certs/config/configuration.py | 7 ++--- sec_certs/dataset/common_criteria.py | 24 ++++++++++------- sec_certs/dataset/cpe.py | 16 +++++------ sec_certs/dataset/cve.py | 23 ++++++++-------- sec_certs/dataset/dataset.py | 20 +++++++------- sec_certs/dataset/fips.py | 13 ++++----- sec_certs/dataset/fips_algorithm.py | 13 ++++----- sec_certs/dataset/protection_profile.py | 6 ++--- sec_certs/helpers.py | 44 +++++++++++++------------------ sec_certs/model/cpe_matching.py | 18 +++++++------ sec_certs/model/dependency_finder.py | 3 ++- sec_certs/model/evaluation.py | 12 ++++----- sec_certs/parallel_processing.py | 7 ++--- sec_certs/sample/cc_maintenance_update.py | 2 +- sec_certs/sample/certificate.py | 14 +++++----- sec_certs/sample/common_criteria.py | 10 +++---- sec_certs/sample/cpe.py | 3 ++- sec_certs/sample/cve.py | 4 +-- sec_certs/sample/fips.py | 21 ++++++++------- sec_certs/sample/protection_profile.py | 7 +++-- sec_certs/serialization/json.py | 4 +-- setup.py | 2 +- tests/fips_test_utils.py | 2 +- tests/test_cc_heuristics.py | 16 +++++------ tests/test_cc_oop.py | 18 ++++++------- tests/test_cc_txt_processing.py | 8 +++--- tests/test_fips_oop.py | 11 ++++---- 34 files changed, 193 insertions(+), 184 deletions(-) (limited to 'tests/test_cc_txt_processing.py') diff --git a/cc_cli.py b/cc_cli.py index 33077e06..bd9e586b 100755 --- a/cc_cli.py +++ b/cc_cli.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -from typing import Optional, List -import click -from pathlib import Path import logging import sys from datetime import datetime +from pathlib import Path +from typing import List, Optional + +import click from sec_certs.config.configuration import config from sec_certs.dataset.common_criteria import CCDataset diff --git a/examples/cc_cpe_labeling.py b/examples/cc_cpe_labeling.py index c3ead2ef..89527bbe 100644 --- a/examples/cc_cpe_labeling.py +++ b/examples/cc_cpe_labeling.py @@ -1,10 +1,10 @@ -from datetime import datetime import logging +from datetime import datetime from pathlib import Path -from sec_certs.dataset.common_criteria import CCDataset from sec_certs.config.configuration import config -from sec_certs.model.evaluation import get_validation_dgsts, evaluate +from sec_certs.dataset.common_criteria import CCDataset +from sec_certs.model.evaluation import evaluate, get_validation_dgsts logger = logging.getLogger(__name__) diff --git a/examples/cc_oop_demo.py b/examples/cc_oop_demo.py index a30d63ce..60949cd1 100644 --- a/examples/cc_oop_demo.py +++ b/examples/cc_oop_demo.py @@ -1,9 +1,9 @@ -from pathlib import Path -from datetime import datetime import logging +from datetime import datetime +from pathlib import Path -from sec_certs.dataset.common_criteria import CCDataset from sec_certs.config.configuration import config +from sec_certs.dataset.common_criteria import CCDataset logger = logging.getLogger(__name__) diff --git a/examples/fips_cpe_labeling.py b/examples/fips_cpe_labeling.py index 4aebe902..8b2d99d2 100644 --- a/examples/fips_cpe_labeling.py +++ b/examples/fips_cpe_labeling.py @@ -1,8 +1,9 @@ -from datetime import datetime import logging -from sec_certs.dataset.fips import FIPSDataset +from datetime import datetime from pathlib import Path -from sec_certs.model.evaluation import get_validation_dgsts, evaluate + +from sec_certs.dataset.fips import FIPSDataset +from sec_certs.model.evaluation import evaluate, get_validation_dgsts logger = logging.getLogger(__name__) diff --git a/examples/fips_oop_demo.py b/examples/fips_oop_demo.py index 1f13dc2a..a84b1ac9 100644 --- a/examples/fips_oop_demo.py +++ b/examples/fips_oop_demo.py @@ -1,11 +1,12 @@ -from pathlib import Path -from datetime import datetime import logging +from datetime import datetime +from pathlib import Path + import click + +from sec_certs.config.configuration import config from sec_certs.dataset.fips import FIPSDataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.config.configuration import config - logger = logging.getLogger(__name__) diff --git a/fips_cli.py b/fips_cli.py index 4336a9bb..b1bca9c2 100755 --- a/fips_cli.py +++ b/fips_cli.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -from typing import Optional, List, Set -import click -from pathlib import Path import logging -import sys import os +import sys from datetime import datetime +from pathlib import Path +from typing import List, Optional, Set + +import click -from sec_certs.config.configuration import config, DEFAULT_CONFIG_PATH +from sec_certs.config.configuration import DEFAULT_CONFIG_PATH, config from sec_certs.dataset.fips import FIPSDataset logger = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 2fc133c9..e51b5a9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,8 @@ exclude = ''' ''' [tool.isort] -profile = "black" +profile = "black" +skip = ["certsvenv"] [tool.mypy] plugins = ["numpy.typing.mypy_plugin"] diff --git a/sec_certs/config/configuration.py b/sec_certs/config/configuration.py index cafb2b23..d06f2e41 100644 --- a/sec_certs/config/configuration.py +++ b/sec_certs/config/configuration.py @@ -1,8 +1,9 @@ -import yaml -from typing import Union +import json from pathlib import Path +from typing import Union + import jsonschema -import json +import yaml class Configuration(object): diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py index 68a5d8bf..bc488d17 100644 --- a/sec_certs/dataset/common_criteria.py +++ b/sec_certs/dataset/common_criteria.py @@ -1,28 +1,32 @@ import copy import itertools +import json import locale import shutil import tempfile from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Dict, Iterator, Optional, Set, Union, List, Tuple, Mapping, ClassVar -import json +from typing import (ClassVar, Dict, Iterator, List, Mapping, Optional, Set, + Tuple, Union) import numpy as np import pandas as pd -from bs4 import Tag, BeautifulSoup +from bs4 import BeautifulSoup, Tag -from sec_certs import helpers as helpers, parallel_processing as cert_processing +from sec_certs import helpers as helpers +from sec_certs import parallel_processing as cert_processing +from sec_certs.config.configuration import config from sec_certs.dataset.dataset import Dataset, logger -from sec_certs.serialization.json import ComplexSerializableType, serialize, CustomJSONDecoder -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.certificate import Certificate from sec_certs.dataset.protection_profile import ProtectionProfileDataset -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate -from sec_certs.config.configuration import config from sec_certs.model.dependency_finder import DependencyFinder +from sec_certs.sample.cc_maintenance_update import \ + CommonCriteriaMaintenanceUpdate +from sec_certs.sample.certificate import Certificate +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.protection_profile import ProtectionProfile +from sec_certs.serialization.json import (ComplexSerializableType, + CustomJSONDecoder, serialize) class CCDataset(Dataset, ComplexSerializableType): diff --git a/sec_certs/dataset/cpe.py b/sec_certs/dataset/cpe.py index 8ad514d8..6727f930 100644 --- a/sec_certs/dataset/cpe.py +++ b/sec_certs/dataset/cpe.py @@ -1,19 +1,19 @@ -from dataclasses import dataclass, field -import logging -from typing import List, Dict, Tuple, Set, Union, ClassVar import itertools +import logging import tempfile -from pathlib import Path +import xml.etree.ElementTree as ET import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import ClassVar, Dict, List, Set, Tuple, Union + +import pandas as pd import sec_certs.helpers as helpers -from sec_certs.sample.cpe import CPE from sec_certs.dataset.cve import CVEDataset +from sec_certs.sample.cpe import CPE from sec_certs.serialization.json import ComplexSerializableType, serialize -import pandas as pd -import xml.etree.ElementTree as ET - logger = logging.getLogger(__name__) diff --git a/sec_certs/dataset/cve.py b/sec_certs/dataset/cve.py index e61e49d6..f143cad6 100644 --- a/sec_certs/dataset/cve.py +++ b/sec_certs/dataset/cve.py @@ -1,24 +1,25 @@ -import itertools -from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple, Union, Final, Set import datetime -from pathlib import Path -import tempfile -import zipfile -import logging import glob +import itertools import json +import logging import shutil +import tempfile +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Final, List, Optional, Set, Tuple, Union import pandas as pd -from sec_certs.parallel_processing import process_parallel import sec_certs.constants as constants import sec_certs.helpers as helpers -from sec_certs.sample.cve import CVE -from sec_certs.sample.cpe import CPE -from sec_certs.serialization.json import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder from sec_certs.config.configuration import config +from sec_certs.parallel_processing import process_parallel +from sec_certs.sample.cpe import CPE +from sec_certs.sample.cve import CVE +from sec_certs.serialization.json import (ComplexSerializableType, + CustomJSONDecoder, CustomJSONEncoder) logger = logging.getLogger(__name__) diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py index 9078d30e..63a781fb 100644 --- a/sec_certs/dataset/dataset.py +++ b/sec_certs/dataset/dataset.py @@ -1,26 +1,24 @@ -from datetime import datetime -import logging -from typing import Dict, Collection, Optional, Set, Union, List, Tuple, Mapping, TypeVar - +import itertools import json +import logging from abc import ABC, abstractmethod +from datetime import datetime from pathlib import Path -import itertools +from typing import (Collection, Dict, List, Mapping, Optional, Set, Tuple, + TypeVar, Union) import requests -import sec_certs.helpers as helpers import sec_certs.constants as constants +import sec_certs.helpers as helpers import sec_certs.parallel_processing as cert_processing -from sec_certs.sample.cpe import CPE - -from sec_certs.sample.certificate import Certificate -from sec_certs.serialization.json import ComplexSerializableType from sec_certs.config.configuration import config -from sec_certs.serialization.json import serialize from sec_certs.dataset.cpe import CPEDataset from sec_certs.dataset.cve import CVEDataset from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.sample.certificate import Certificate +from sec_certs.sample.cpe import CPE +from sec_certs.serialization.json import ComplexSerializableType, serialize logger = logging.getLogger(__name__) diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py index 60a0543a..cea42eeb 100644 --- a/sec_certs/dataset/fips.py +++ b/sec_certs/dataset/fips.py @@ -1,22 +1,23 @@ import datetime -import tempfile import logging import os +import tempfile from itertools import groupby from pathlib import Path +from typing import Dict, List, Mapping, Optional, Set, Tuple -from typing import Set, Tuple, List, Dict, Optional, Mapping from bs4 import BeautifulSoup, NavigableString from graphviz import Digraph -from sec_certs import constants as constants, parallel_processing as cert_processing, helpers as helpers +from sec_certs import constants as constants +from sec_certs import helpers as helpers +from sec_certs import parallel_processing as cert_processing from sec_certs.config.configuration import config -from sec_certs.sample.certificate import Certificate from sec_certs.dataset.dataset import Dataset, logger from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.serialization.json import ComplexSerializableType, serialize +from sec_certs.sample.certificate import Certificate from sec_certs.sample.fips import FIPSCertificate - +from sec_certs.serialization.json import ComplexSerializableType, serialize logger = logging.getLogger(__name__) diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py index e4fbc987..a6ec381e 100644 --- a/sec_certs/dataset/fips_algorithm.py +++ b/sec_certs/dataset/fips_algorithm.py @@ -1,17 +1,18 @@ import json import logging from pathlib import Path -from typing import Dict, Union, List +from typing import Dict, List, Union from bs4 import BeautifulSoup -from sec_certs import helpers as helpers, constants as constants, parallel_processing as cert_processing +from sec_certs import constants as constants +from sec_certs import helpers as helpers +from sec_certs import parallel_processing as cert_processing +from sec_certs.config.configuration import config from sec_certs.dataset.dataset import Dataset -from sec_certs.serialization.json import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder - from sec_certs.sample.fips import FIPSCertificate -from sec_certs.config.configuration import config - +from sec_certs.serialization.json import (ComplexSerializableType, + CustomJSONDecoder, CustomJSONEncoder) logger = logging.getLogger(__name__) diff --git a/sec_certs/dataset/protection_profile.py b/sec_certs/dataset/protection_profile.py index 6a3afecc..bab82fb0 100644 --- a/sec_certs/dataset/protection_profile.py +++ b/sec_certs/dataset/protection_profile.py @@ -1,9 +1,9 @@ -from dataclasses import dataclass -from typing import Dict, Tuple, Union, Optional, ClassVar -from pathlib import Path import json import logging import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import ClassVar, Dict, Optional, Tuple, Union import sec_certs.helpers as helpers from sec_certs.sample.protection_profile import ProtectionProfile diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index b6cdff84..60775915 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -1,40 +1,34 @@ +import copy +import hashlib +import html +import logging import os import re -from typing import Sequence, Tuple, Optional, Set, List, Dict, Hashable, Any -import logging -import pikepdf -import requests +import subprocess +import time +from datetime import date +from enum import Enum from multiprocessing.pool import ThreadPool from pathlib import Path -from tqdm import tqdm as tqdm_original -import hashlib -import html -from typing import Union, List -from datetime import date +from typing import (Any, Dict, Hashable, List, Optional, Sequence, Set, Tuple, + Union) + +import matplotlib.pyplot as plt import numpy as np import pandas as pd -import subprocess -import time -import copy +import pikepdf +import requests from packaging.version import VERSION_PATTERN - - -from enum import Enum -import matplotlib.pyplot as plt from PyPDF2 import PdfFileReader +from tqdm import tqdm as tqdm_original -import sec_certs.constants import sec_certs.constants as constants -from sec_certs.config.configuration import config from sec_certs.cert_rules import REGEXEC_SEP from sec_certs.cert_rules import rules as cc_search_rules -from sec_certs.constants import ( - TAG_MATCH_COUNTER, - APPEND_DETAILED_MATCH_MATCHES, - TAG_MATCH_MATCHES, - FILE_ERRORS_STRATEGY, - LINE_SEPARATOR, -) +from sec_certs.config.configuration import config +from sec_certs.constants import (APPEND_DETAILED_MATCH_MATCHES, + FILE_ERRORS_STRATEGY, LINE_SEPARATOR, + TAG_MATCH_COUNTER, TAG_MATCH_MATCHES) logger = logging.getLogger(__name__) diff --git a/sec_certs/model/cpe_matching.py b/sec_certs/model/cpe_matching.py index dae6c558..042f6705 100644 --- a/sec_certs/model/cpe_matching.py +++ b/sec_certs/model/cpe_matching.py @@ -1,15 +1,17 @@ -from sklearn.base import BaseEstimator -from typing import Dict, Tuple, Set, List, Optional, Union -from sec_certs.sample.cpe import CPE -import sec_certs.helpers as helpers import itertools -import re -from rapidfuzz import process, fuzz -import operator -from pathlib import Path import json import logging +import operator +import re +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple, Union + from packaging.version import parse +from rapidfuzz import fuzz, process +from sklearn.base import BaseEstimator + +import sec_certs.helpers as helpers +from sec_certs.sample.cpe import CPE logger = logging.getLogger(__name__) diff --git a/sec_certs/model/dependency_finder.py b/sec_certs/model/dependency_finder.py index 6afcf7ef..07ccb199 100644 --- a/sec_certs/model/dependency_finder.py +++ b/sec_certs/model/dependency_finder.py @@ -1,4 +1,5 @@ -from typing import List, Set, Dict, Tuple, Union, Optional +from typing import Dict, List, Optional, Set, Tuple, Union + from sec_certs.sample.common_criteria import CommonCriteriaCert Certificates = Dict[str, CommonCriteriaCert] diff --git a/sec_certs/model/evaluation.py b/sec_certs/model/evaluation.py index 4eaeea6a..0ef23dd9 100644 --- a/sec_certs/model/evaluation.py +++ b/sec_certs/model/evaluation.py @@ -1,15 +1,15 @@ import json -from pathlib import Path import logging -from typing import List, Set, Union, Optional +from pathlib import Path +from typing import List, Optional, Set, Union +import numpy as np + +import sec_certs.helpers as helpers +from sec_certs.dataset.cpe import CPEDataset from sec_certs.sample.common_criteria import CommonCriteriaCert from sec_certs.sample.fips import FIPSCertificate -from sec_certs.dataset.cpe import CPEDataset from sec_certs.serialization.json import CustomJSONEncoder -import sec_certs.helpers as helpers - -import numpy as np logger = logging.getLogger(__name__) diff --git a/sec_certs/parallel_processing.py b/sec_certs/parallel_processing.py index 28d94914..3bfa2826 100644 --- a/sec_certs/parallel_processing.py +++ b/sec_certs/parallel_processing.py @@ -1,8 +1,9 @@ -from sec_certs.helpers import tqdm +import time from multiprocessing.pool import ThreadPool -from billiard.pool import Pool from typing import Callable, Iterable, Optional, Union -import time + +from billiard.pool import Pool +from sec_certs.helpers import tqdm def process_parallel( diff --git a/sec_certs/sample/cc_maintenance_update.py b/sec_certs/sample/cc_maintenance_update.py index 2ed939c7..fa4012e3 100644 --- a/sec_certs/sample/cc_maintenance_update.py +++ b/sec_certs/sample/cc_maintenance_update.py @@ -1,6 +1,6 @@ import logging -from typing import Optional, Dict, List, ClassVar, Tuple from datetime import date +from typing import ClassVar, Dict, List, Optional, Tuple import sec_certs.helpers as helpers from sec_certs.sample.common_criteria import CommonCriteriaCert diff --git a/sec_certs/sample/certificate.py b/sec_certs/sample/certificate.py index 14c77f2e..991e86a6 100644 --- a/sec_certs/sample/certificate.py +++ b/sec_certs/sample/certificate.py @@ -1,15 +1,15 @@ -import logging -from pathlib import Path import copy -import json import itertools - +import json +import logging from abc import ABC, abstractmethod -from typing import Optional, Union, TypeVar, Type, Any +from pathlib import Path +from typing import Any, Optional, Type, TypeVar, Union -from sec_certs.serialization.json import CustomJSONDecoder, CustomJSONEncoder, ComplexSerializableType -from sec_certs.model.cpe_matching import CPEClassifier from sec_certs.dataset.cve import CVEDataset +from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.serialization.json import (ComplexSerializableType, + CustomJSONDecoder, CustomJSONEncoder) logger = logging.getLogger(__name__) diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py index 657477d7..a0c41d13 100644 --- a/sec_certs/sample/common_criteria.py +++ b/sec_certs/sample/common_criteria.py @@ -3,18 +3,18 @@ import operator from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path -from typing import Optional, List, Dict, Tuple, Union, Any, Set, ClassVar - +from typing import Any, ClassVar, Dict, List, Optional, Set, Tuple, Union import requests from bs4 import Tag -from sec_certs import helpers, constants as constants +from sec_certs import constants as constants +from sec_certs import helpers +from sec_certs.model.cpe_matching import CPEClassifier from sec_certs.sample.certificate import Certificate, logger +from sec_certs.sample.protection_profile import ProtectionProfile from sec_certs.serialization.json import ComplexSerializableType from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.sample.protection_profile import ProtectionProfile -from sec_certs.model.cpe_matching import CPEClassifier class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializableType): diff --git a/sec_certs/sample/cpe.py b/sec_certs/sample/cpe.py index a67eacbb..1099c144 100644 --- a/sec_certs/sample/cpe.py +++ b/sec_certs/sample/cpe.py @@ -1,5 +1,6 @@ from dataclasses import dataclass -from typing import ClassVar, List, Optional, Tuple, Dict +from typing import ClassVar, Dict, List, Optional, Tuple + from sec_certs.serialization.json import ComplexSerializableType from sec_certs.serialization.pandas import PandasSerializableType diff --git a/sec_certs/sample/cve.py b/sec_certs/sample/cve.py index 58edf1e2..94484374 100644 --- a/sec_certs/sample/cve.py +++ b/sec_certs/sample/cve.py @@ -1,13 +1,13 @@ import datetime import itertools from dataclasses import dataclass -from typing import Any, Dict, List, Optional, ClassVar, Tuple +from typing import Any, ClassVar, Dict, List, Optional, Tuple from dateutil.parser import isoparse +from sec_certs.sample.cpe import CPE from sec_certs.serialization.json import ComplexSerializableType from sec_certs.serialization.pandas import PandasSerializableType -from sec_certs.sample.cpe import CPE @dataclass(init=False) diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py index 57082218..3a3d2e5a 100644 --- a/sec_certs/sample/fips.py +++ b/sec_certs/sample/fips.py @@ -3,25 +3,26 @@ import re from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import ClassVar, Dict, Optional, Union, List, Tuple, Set, Pattern +from typing import ClassVar, Dict, List, Optional, Pattern, Set, Tuple, Union import requests -from bs4 import Tag, NavigableString, BeautifulSoup +from bs4 import BeautifulSoup, NavigableString, Tag from dateutil import parser from tabula import read_pdf -import sec_certs.constants -from sec_certs import helpers, constants as constants -from sec_certs.cert_rules import fips_common_rules, REGEXEC_SEP, fips_rules - -from sec_certs.sample.certificate import Certificate, logger +import sec_certs.constants as constants +from sec_certs import constants as constants +from sec_certs import helpers +from sec_certs.cert_rules import REGEXEC_SEP, fips_common_rules, fips_rules from sec_certs.config.configuration import config from sec_certs.constants import LINE_SEPARATOR -from sec_certs.helpers import save_modified_cert_file, normalize_match_string, load_cert_file -from sec_certs.serialization.json import ComplexSerializableType from sec_certs.dataset.cpe import CPEDataset -from sec_certs.sample.cpe import CPE +from sec_certs.helpers import (load_cert_file, normalize_match_string, + save_modified_cert_file) from sec_certs.model.cpe_matching import CPEClassifier +from sec_certs.sample.certificate import Certificate, logger +from sec_certs.sample.cpe import CPE +from sec_certs.serialization.json import ComplexSerializableType class FIPSCertificate(Certificate, ComplexSerializableType): diff --git a/sec_certs/sample/protection_profile.py b/sec_certs/sample/protection_profile.py index 05dab814..6ce2d00e 100644 --- a/sec_certs/sample/protection_profile.py +++ b/sec_certs/sample/protection_profile.py @@ -1,11 +1,10 @@ -from dataclasses import dataclass -from typing import Optional, FrozenSet import copy -from typing import Dict import logging +from dataclasses import dataclass +from typing import Dict, FrozenSet, Optional -from sec_certs.serialization.json import ComplexSerializableType import sec_certs.helpers as helpers +from sec_certs.serialization.json import ComplexSerializableType logger = logging.getLogger(__name__) diff --git a/sec_certs/serialization/json.py b/sec_certs/serialization/json.py index 2ab5aa93..8c229f65 100644 --- a/sec_certs/serialization/json.py +++ b/sec_certs/serialization/json.py @@ -1,8 +1,8 @@ +import copy import json from datetime import date from pathlib import Path -from typing import Dict, List, Union, Optional, Callable -import copy +from typing import Callable, Dict, List, Optional, Union class ComplexSerializableType: diff --git a/setup.py b/setup.py index 595e3811..a34108f8 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from setuptools import setup, find_packages +from setuptools import find_packages, setup with open("requirements.txt") as f: requirements = f.read().splitlines() diff --git a/tests/fips_test_utils.py b/tests/fips_test_utils.py index 07734d9d..89b61e7e 100644 --- a/tests/fips_test_utils.py +++ b/tests/fips_test_utils.py @@ -1,5 +1,5 @@ -from typing import List from pathlib import Path +from typing import List def generate_html(ids: List[str], path: Path): diff --git a/tests/test_cc_heuristics.py b/tests/test_cc_heuristics.py index 527d6e6f..e2dd7855 100644 --- a/tests/test_cc_heuristics.py +++ b/tests/test_cc_heuristics.py @@ -1,19 +1,19 @@ import copy import datetime +import shutil import tempfile +from pathlib import Path +from typing import ClassVar, Dict from unittest import TestCase + +import tests.data.test_cc_heuristics from sec_certs.dataset.common_criteria import CCDataset -from sec_certs.sample.common_criteria import CommonCriteriaCert -from sec_certs.sample.protection_profile import ProtectionProfile from sec_certs.dataset.cpe import CPEDataset -from sec_certs.sample.cpe import CPE from sec_certs.dataset.cve import CVEDataset +from sec_certs.sample.common_criteria import CommonCriteriaCert +from sec_certs.sample.cpe import CPE from sec_certs.sample.cve import CVE -from pathlib import Path -from typing import ClassVar, Dict -import shutil - -import tests.data.test_cc_heuristics +from sec_certs.sample.protection_profile import ProtectionProfile class TestCommonCriteriaHeuristics(TestCase): diff --git a/tests/test_cc_oop.py b/tests/test_cc_oop.py index 9a598bc5..0149a23d 100644 --- a/tests/test_cc_oop.py +++ b/tests/test_cc_oop.py @@ -1,18 +1,18 @@ -import tempfile -from unittest import TestCase -from pathlib import Path -from tempfile import TemporaryDirectory, mkstemp, NamedTemporaryFile -from datetime import date, datetime -import json import filecmp -import shutil +import json import os +import shutil +import tempfile +from datetime import date, datetime +from pathlib import Path +from tempfile import NamedTemporaryFile, TemporaryDirectory, mkstemp +from unittest import TestCase +import sec_certs.constants as constants +import sec_certs.helpers as helpers from sec_certs.dataset.common_criteria import CCDataset from sec_certs.sample.common_criteria import CommonCriteriaCert from sec_certs.sample.protection_profile import ProtectionProfile -import sec_certs.helpers as helpers -import sec_certs.constants as constants class TestCommonCriteriaOOP(TestCase): diff --git a/tests/test_cc_txt_processing.py b/tests/test_cc_txt_processing.py index 96716fbe..d33fa6f6 100644 --- a/tests/test_cc_txt_processing.py +++ b/tests/test_cc_txt_processing.py @@ -1,10 +1,10 @@ -import tests.data.test_cc_heuristics -from unittest import TestCase +import shutil +import tempfile from pathlib import Path from typing import ClassVar -import tempfile -import shutil +from unittest import TestCase +import tests.data.test_cc_heuristics from sec_certs.dataset.common_criteria import CCDataset from sec_certs.sample.common_criteria import CommonCriteriaCert diff --git a/tests/test_fips_oop.py b/tests/test_fips_oop.py index c39558b6..74651acf 100644 --- a/tests/test_fips_oop.py +++ b/tests/test_fips_oop.py @@ -1,16 +1,15 @@ -from unittest import TestCase +import shutil from pathlib import Path from tempfile import TemporaryDirectory -from typing import Optional, Union, ClassVar, Final, List, Dict -import shutil +from typing import ClassVar, Dict, Final, List, Optional, Union +from unittest import TestCase +import tests.data.test_fips_oop +from sec_certs.config.configuration import config from sec_certs.dataset.fips import FIPSDataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.config.configuration import config from tests.fips_test_utils import generate_html -import tests.data.test_fips_oop - def _set_up_dataset(td, certs): dataset = FIPSDataset({}, Path(td), "test_dataset", "fips_test_dataset") -- cgit v1.3.1 From 4f255665d2c335185fb68e288801c89c66dc5a5a Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sat, 18 Dec 2021 17:50:51 +0100 Subject: Fix MyPy errors --- sec_certs/dataset/common_criteria.py | 8 ++++---- sec_certs/dataset/dataset.py | 4 ++-- sec_certs/dataset/protection_profile.py | 6 +++--- sec_certs/helpers.py | 13 +++++-------- sec_certs/sample/common_criteria.py | 6 +++++- sec_certs/sample/cpe.py | 4 ++-- sec_certs/sample/fips.py | 4 ++-- sec_certs/sample/protection_profile.py | 2 +- tests/test_cc_heuristics.py | 14 ++++++++++---- tests/test_cc_txt_processing.py | 10 ++++++---- tests/test_fips_oop.py | 2 +- 11 files changed, 41 insertions(+), 32 deletions(-) (limited to 'tests/test_cc_txt_processing.py') diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py index a781db41..8fe2b710 100644 --- a/sec_certs/dataset/common_criteria.py +++ b/sec_certs/dataset/common_criteria.py @@ -7,7 +7,7 @@ import tempfile from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import ClassVar, Dict, Iterator, List, Mapping, Optional, Set, Tuple, Union +from typing import ClassVar, Dict, Iterator, List, Mapping, Optional, Set, Tuple, Union, Callable import numpy as np import pandas as pd @@ -217,10 +217,10 @@ class CCDataset(Dataset, ComplexSerializableType): @serialize def process_protection_profiles(self, to_download: bool = True, keep_metadata: bool = True): logger.info("Processing protection profiles.") - constructor = {True: ProtectionProfileDataset.from_web, False: ProtectionProfileDataset.from_json} + constructor: Dict[bool, Callable[..., ProtectionProfileDataset]] = {True: ProtectionProfileDataset.from_web, False: ProtectionProfileDataset.from_json} if to_download is True and not self.auxillary_datasets_dir.exists(): self.auxillary_datasets_dir.mkdir() - pp_dataset: ProtectionProfileDataset = constructor[to_download](self.pp_dataset_path) + pp_dataset = constructor[to_download](self.pp_dataset_path) for cert in self: if cert.protection_profiles is None: @@ -342,7 +342,7 @@ class CCDataset(Dataset, ComplexSerializableType): df_main = df_main.drop_duplicates() profiles = { - x.dgst: set([ProtectionProfile(y) for y in helpers.sanitize_protection_profiles(x.protection_profiles)]) + x.dgst: set([ProtectionProfile(pp_name=y) for y in helpers.sanitize_protection_profiles(x.protection_profiles)]) for x in df_base.itertuples() } updates: Dict[str, Set] = {x.dgst: set() for x in df_base.itertuples()} diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py index e6a4396d..bf129a82 100644 --- a/sec_certs/dataset/dataset.py +++ b/sec_certs/dataset/dataset.py @@ -4,7 +4,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime from pathlib import Path -from typing import Collection, Dict, List, Mapping, Optional, Set, Tuple, TypeVar, Union +from typing import Collection, Dict, List, Mapping, Optional, Set, Tuple, TypeVar, Union, Type import requests @@ -119,7 +119,7 @@ class Dataset(ABC): return dset @classmethod - def from_json(cls: T, input_path: Union[str, Path]) -> T: + def from_json(cls: Type[T], input_path: Union[str, Path]) -> T: dset = ComplexSerializableType.from_json(input_path) dset.root_dir = Path(input_path).parent.absolute() dset.set_local_paths() diff --git a/sec_certs/dataset/protection_profile.py b/sec_certs/dataset/protection_profile.py index bab82fb0..4e7ba747 100644 --- a/sec_certs/dataset/protection_profile.py +++ b/sec_certs/dataset/protection_profile.py @@ -15,15 +15,15 @@ logger = logging.getLogger(__name__) class ProtectionProfileDataset: static_dataset_url: ClassVar[str] = "https://ajanovsky.cz/pp_data_complete_processed.json" - pps: Dict[Tuple[str, str], ProtectionProfile] + pps: Dict[Tuple[str, Optional[str]], ProtectionProfile] def __iter__(self): yield from self.pps.values() - def __getitem__(self, item: Tuple[str, str]) -> ProtectionProfile: + def __getitem__(self, item: Tuple[str, Optional[str]]) -> ProtectionProfile: return self.pps.__getitem__(item) - def __setitem__(self, key: Tuple[str, str], value: ProtectionProfile): + def __setitem__(self, key: Tuple[str, Optional[str]], value: ProtectionProfile): self.pps.__setitem__(key, value) def __contains__(self, key): diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index bf96eaa8..9198f5c5 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -100,13 +100,10 @@ def sanitize_date(record: Union[pd.Timestamp, date, np.datetime64]) -> Union[dat return record # type: ignore -def sanitize_string(record: Optional[str]) -> Optional[str]: - if not record: - return None - else: - # TODO: There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I & J' that has to be unescaped twice - string = html.unescape(html.unescape(record)).replace("\n", "") - return " ".join(string.split()) +def sanitize_string(record: str) -> str: + # TODO: There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT / AT90SC12836RCFT rev. I & J' that has to be unescaped twice + string = html.unescape(html.unescape(record)).replace("\n", "") + return " ".join(string.split()) def sanitize_security_levels(record: Union[str, set]) -> set: @@ -593,7 +590,7 @@ def search_only_headers_bsi(filepath: Path): def extract_keywords(filepath: Path) -> Tuple[str, Optional[Dict[str, Dict[str, int]]]]: try: - result = parse_cert_file(filepath, cc_search_rules, -1, sec_certs.constants.LINE_SEPARATOR)[0] + result = parse_cert_file(filepath, cc_search_rules, -1, constants.LINE_SEPARATOR)[0] processed_result = {} top_level_keys = list(result.keys()) diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py index a0c41d13..994f56ca 100644 --- a/sec_certs/sample/common_criteria.py +++ b/sec_certs/sample/common_criteria.py @@ -256,7 +256,11 @@ class CommonCriteriaCert(Certificate, PandasSerializableType, ComplexSerializabl self.status = status self.category = category self.name = helpers.sanitize_string(name) - self.manufacturer = helpers.sanitize_string(manufacturer) + + self.manufacturer = None + if manufacturer: + self.manufacturer = helpers.sanitize_string(manufacturer) + self.scheme = scheme self.security_level = helpers.sanitize_security_levels(security_level) self.not_valid_before = helpers.sanitize_date(not_valid_before) diff --git a/sec_certs/sample/cpe.py b/sec_certs/sample/cpe.py index 1099c144..aeab6dc9 100644 --- a/sec_certs/sample/cpe.py +++ b/sec_certs/sample/cpe.py @@ -7,7 +7,7 @@ from sec_certs.serialization.pandas import PandasSerializableType @dataclass(init=False) class CPE(PandasSerializableType, ComplexSerializableType): - uri: Optional[str] + uri: str title: Optional[str] version: Optional[str] vendor: Optional[str] @@ -27,7 +27,7 @@ class CPE(PandasSerializableType, ComplexSerializableType): def __init__( self, - uri: Optional[str] = None, + uri: str, title: Optional[str] = None, start_version: Optional[Tuple[str, str]] = None, end_version: Optional[Tuple[str, str]] = None, diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py index fe9665fc..01ac3f2d 100644 --- a/sec_certs/sample/fips.py +++ b/sec_certs/sample/fips.py @@ -636,7 +636,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): if match not in items_found[rule_str]: items_found[rule_str][match] = {} items_found[rule_str][match][constants.TAG_MATCH_COUNTER] = 0 - if sec_certs.constants.APPEND_DETAILED_MATCH_MATCHES: + if constants.APPEND_DETAILED_MATCH_MATCHES: items_found[rule_str][match][constants.TAG_MATCH_MATCHES] = [] # else: # items_found[rule_str][match][TAG_MATCH_MATCHES] = ['List of matches positions disabled. Set APPEND_DETAILED_MATCH_MATCHES to True'] @@ -647,7 +647,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): # line_number = get_line_number(lines, line_length_compensation, match_span[0]) # start index, end index, line number # items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number]) - if sec_certs.constants.APPEND_DETAILED_MATCH_MATCHES: + if constants.APPEND_DETAILED_MATCH_MATCHES: items_found[rule_str][match][constants.TAG_MATCH_MATCHES].append([match_span[0], match_span[1]]) # highlight all found strings (by xxxxx) from the input text and store the rest diff --git a/sec_certs/sample/protection_profile.py b/sec_certs/sample/protection_profile.py index 6ce2d00e..d8825ced 100644 --- a/sec_certs/sample/protection_profile.py +++ b/sec_certs/sample/protection_profile.py @@ -15,7 +15,7 @@ class ProtectionProfile(ComplexSerializableType): Object for holding protection profiles. """ - pp_name: Optional[str] + pp_name: str pp_link: Optional[str] = None pp_ids: Optional[FrozenSet[str]] = None diff --git a/tests/test_cc_heuristics.py b/tests/test_cc_heuristics.py index e2dd7855..13bff947 100644 --- a/tests/test_cc_heuristics.py +++ b/tests/test_cc_heuristics.py @@ -3,7 +3,7 @@ import datetime import shutil import tempfile from pathlib import Path -from typing import ClassVar, Dict +from typing import ClassVar, Dict, List from unittest import TestCase import tests.data.test_cc_heuristics @@ -17,15 +17,21 @@ from sec_certs.sample.protection_profile import ProtectionProfile class TestCommonCriteriaHeuristics(TestCase): - dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / "vulnerable_dataset.json" + dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / "vulnerable_dataset.json" # type: ignore # mypy issue #1422 data_dir_path: ClassVar[Path] = dataset_json_path.parent + tmp_dir: ClassVar[tempfile.TemporaryDirectory] + cc_dset: CCDataset + cve_dset: CVEDataset + cves: List[CVE] + cpe_dset: CPEDataset + cpes: List[CPE] @classmethod def setUpClass(cls) -> None: - cls.tmp_dir: ClassVar[tempfile.TemporaryDirectory] = tempfile.TemporaryDirectory() + cls.tmp_dir = tempfile.TemporaryDirectory() shutil.copytree(cls.data_dir_path, cls.tmp_dir.name, dirs_exist_ok=True) - cls.cc_dset: CCDataset = CCDataset.from_json(Path(cls.tmp_dir.name) / "vulnerable_dataset.json") + cls.cc_dset = CCDataset.from_json(Path(cls.tmp_dir.name) / "vulnerable_dataset.json") cls.cc_dset.process_protection_profiles() cls.cc_dset.download_all_pdfs() cls.cc_dset.convert_all_pdfs() diff --git a/tests/test_cc_txt_processing.py b/tests/test_cc_txt_processing.py index d33fa6f6..cc1cb23c 100644 --- a/tests/test_cc_txt_processing.py +++ b/tests/test_cc_txt_processing.py @@ -9,16 +9,18 @@ from sec_certs.dataset.common_criteria import CCDataset from sec_certs.sample.common_criteria import CommonCriteriaCert -class TestCommonCriteriaHeuristics(TestCase): - dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / "vulnerable_dataset.json" +class TestCommonCriteriaTextProcessing(TestCase): + dataset_json_path: ClassVar[Path] = Path(tests.data.test_cc_heuristics.__path__[0]) / "vulnerable_dataset.json" # type: ignore # mypy issue #1422 data_dir_path: ClassVar[Path] = dataset_json_path.parent + tmp_dir: ClassVar[tempfile.TemporaryDirectory] + cc_dset: CCDataset @classmethod def setUpClass(cls) -> None: - cls.tmp_dir: ClassVar[tempfile.TemporaryDirectory] = tempfile.TemporaryDirectory() + cls.tmp_dir = tempfile.TemporaryDirectory() shutil.copytree(cls.data_dir_path, cls.tmp_dir.name, dirs_exist_ok=True) - cls.cc_dset: CCDataset = CCDataset.from_json(Path(cls.tmp_dir.name) / "vulnerable_dataset.json") + cls.cc_dset = CCDataset.from_json(Path(cls.tmp_dir.name) / "vulnerable_dataset.json") cls.cc_dset.download_all_pdfs() cls.cc_dset.convert_all_pdfs() diff --git a/tests/test_fips_oop.py b/tests/test_fips_oop.py index 74651acf..9eb04d67 100644 --- a/tests/test_fips_oop.py +++ b/tests/test_fips_oop.py @@ -18,7 +18,7 @@ def _set_up_dataset(td, certs): return dataset -def _set_up_dataset_for_full(td, certs, cpe_dset_path: Union[str, Path] = None, cve_dset_path: Union[str, Path] = None): +def _set_up_dataset_for_full(td, certs, cpe_dset_path: Path, cve_dset_path: Path): dataset = _set_up_dataset(td, certs) dataset.auxillary_datasets_dir.mkdir(exist_ok=True) -- cgit v1.3.1