From bc1e5d61cea6597a184f79baed8caedab35fb66e Mon Sep 17 00:00:00 2001 From: J08nY Date: Thu, 22 Apr 2021 20:52:13 +0200 Subject: Split MIP+IUT code into CLI part in examples and rest in library. --- examples/fips_iut_demo.py | 29 +++++++++++++++++++++++++++++ examples/fips_mip_demo.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100755 examples/fips_iut_demo.py create mode 100755 examples/fips_mip_demo.py (limited to 'examples') diff --git a/examples/fips_iut_demo.py b/examples/fips_iut_demo.py new file mode 100755 index 00000000..fc10286e --- /dev/null +++ b/examples/fips_iut_demo.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import click +from sec_certs.iut import IUTDataset + + +@click.command() +@click.argument("directory", type=click.Path(exists=True, file_okay=False)) +@click.argument("output", type=click.Path(dir_okay=False, writable=True)) +def main(directory, output): + """ + Parse FIPS 'Implementation Under Test' pages. + + \b + To use, download pages from the URL: + https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/IUT-List + into a directory `d` and name them `fips_iut_.html`. + + \b + Then run: + in_process.py fips-iut d output.json + to obtain the parsed output in `output.json`. + """ + dataset = IUTDataset.from_dump(directory) + dataset.to_json(output) + + +if __name__ == "__main__": + main() diff --git a/examples/fips_mip_demo.py b/examples/fips_mip_demo.py new file mode 100755 index 00000000..0173efda --- /dev/null +++ b/examples/fips_mip_demo.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import click +from sec_certs.mip import MIPDataset + + +@click.command() +@click.argument("directory", type=click.Path(exists=True, file_okay=False)) +@click.argument("output", type=click.Path(dir_okay=False, writable=True)) +def main(directory, output): + """ + Parse FIPS 'Modules In Process' pages. + + \b + To use, download pages from the URL: + https://csrc.nist.gov/Projects/cryptographic-module-validation-program/modules-in-process/Modules-In-Process-List + into a directory `d` and name them `fips_mip_.html`. + + \b + Then run: + in_process.py fips-mip d output.json + to obtain the parsed output in `output.json`. + """ + dataset = MIPDataset.from_dump(directory) + dataset.to_json(output) + + +if __name__ == "__main__": + main() -- cgit v1.3.1 From cf60066621b92ec9580de10e0c0835a8a0aa4a05 Mon Sep 17 00:00:00 2001 From: J08nY Date: Thu, 23 Dec 2021 18:12:45 +0100 Subject: Split sample and dataset stuff in IUT and MIP. --- examples/fips_iut_demo.py | 2 +- examples/fips_mip_demo.py | 2 +- sec_certs/dataset/fips_iut.py | 86 ++++++++++++++++++++++ sec_certs/dataset/fips_mip.py | 109 ++++++++++++++++++++++++++++ sec_certs/iut.py | 128 --------------------------------- sec_certs/mip.py | 161 ------------------------------------------ sec_certs/sample/fips_iut.py | 47 ++++++++++++ sec_certs/sample/fips_mip.py | 55 +++++++++++++++ 8 files changed, 299 insertions(+), 291 deletions(-) create mode 100644 sec_certs/dataset/fips_iut.py create mode 100644 sec_certs/dataset/fips_mip.py delete mode 100644 sec_certs/iut.py delete mode 100644 sec_certs/mip.py create mode 100644 sec_certs/sample/fips_iut.py create mode 100644 sec_certs/sample/fips_mip.py (limited to 'examples') diff --git a/examples/fips_iut_demo.py b/examples/fips_iut_demo.py index fc10286e..aaeaa103 100755 --- a/examples/fips_iut_demo.py +++ b/examples/fips_iut_demo.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import click -from sec_certs.iut import IUTDataset +from sec_certs.dataset.fips_iut import IUTDataset @click.command() diff --git a/examples/fips_mip_demo.py b/examples/fips_mip_demo.py index 0173efda..bff6f3a1 100755 --- a/examples/fips_mip_demo.py +++ b/examples/fips_mip_demo.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import click -from sec_certs.mip import MIPDataset +from sec_certs.dataset.fips_mip import MIPDataset @click.command() diff --git a/sec_certs/dataset/fips_iut.py b/sec_certs/dataset/fips_iut.py new file mode 100644 index 00000000..183c8848 --- /dev/null +++ b/sec_certs/dataset/fips_iut.py @@ -0,0 +1,86 @@ +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import List, Union, Mapping + +from bs4 import BeautifulSoup, Tag +from tqdm import tqdm + +from sec_certs.helpers import to_utc +from sec_certs.dataset.dataset import logger +from sec_certs.sample.fips_iut import IUTEntry, IUTSnapshot +from sec_certs.serialization.json import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder + + +@dataclass +class IUTDataset(ComplexSerializableType): + snapshots: List[IUTSnapshot] + + def __iter__(self): + yield from self.snapshots + + def __getitem__(self, item: int) -> IUTSnapshot: + return self.snapshots.__getitem__(item) + + def __len__(self) -> int: + return len(self.snapshots) + + @classmethod + def from_dump(cls, dump_path: Union[str, Path]) -> "IUTDataset": + directory = Path(dump_path) + snapshots = [] + fnames = list(directory.glob("*")) + for fname in tqdm(sorted(fnames), total=len(fnames)): + snapshot_date = to_utc( + datetime.fromisoformat(fname.name[len("fips_iut_") : -len(".html")]) + ) + with open(fname) as f: + soup = BeautifulSoup(f, "html.parser") + tables = soup.find_all("table") + if len(tables) != 1: + logger.error(f"*** Not only a single table in {fname}.") + continue + last_updated_elem = next( + filter( + lambda e: isinstance(e, Tag) and e.name == "p", + soup.find(id="content").next_siblings, + ) + ) + last_updated_text = str(last_updated_elem.string).strip() + last_updated = datetime.strptime( + last_updated_text, "Last Updated: %m/%d/%Y" + ).date() + table = tables[0].find("tbody") + lines = table.find_all("tr") + entries = { + IUTEntry( + str(line[0].string), + str(line[1].string), + str(line[2].string), + datetime.strptime(str(line[3].string), "%m/%d/%Y").date(), + ) + for line in map(lambda tr: tr.find_all("td"), lines) + } + snapshots.append(IUTSnapshot(entries, snapshot_date, last_updated)) + return cls(snapshots) + + def to_dict(self): + return { + "snapshots": list(self.snapshots) + } + + @classmethod + def from_dict(cls, dct: Mapping) -> "IUTDataset": + return cls( + dct["snapshots"] + ) + + def to_json(self, json_path: Union[str, Path]): + with open(json_path, 'w') as handle: + json.dump(self, handle, indent=4, cls=CustomJSONEncoder) + + @classmethod + def from_json(cls, json_path: Union[str, Path]) -> "IUTDataset": + with Path(json_path).open('r') as handle: + return json.load(handle, cls=CustomJSONDecoder) diff --git a/sec_certs/dataset/fips_mip.py b/sec_certs/dataset/fips_mip.py new file mode 100644 index 00000000..e0852bc2 --- /dev/null +++ b/sec_certs/dataset/fips_mip.py @@ -0,0 +1,109 @@ +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import List, Union, Mapping + +from bs4 import BeautifulSoup, Tag +from tqdm import tqdm + +from sec_certs.dataset.dataset import logger +from sec_certs.helpers import to_utc +from sec_certs.sample.fips_mip import MIPEntry, MIPSnapshot, MIPStatus +from sec_certs.serialization.json import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder + + +@dataclass +class MIPDataset(ComplexSerializableType): + snapshots: List[MIPSnapshot] + + def __iter__(self): + yield from self.snapshots + + def __getitem__(self, item: int) -> MIPSnapshot: + return self.snapshots.__getitem__(item) + + def __len__(self) -> int: + return len(self.snapshots) + + @classmethod + def from_dump(cls, dump_path: Union[str, Path]) -> "MIPDataset": + directory = Path(dump_path) + snapshots = [] + fnames = list(directory.glob("*")) + for fname in tqdm(sorted(fnames), total=len(fnames)): + snapshot_date = to_utc( + datetime.fromisoformat(fname.name[len("fips_mip_") : -len(".html")]) + ) + with open(fname) as f: + soup = BeautifulSoup(f, "html.parser") + tables = soup.find_all("table") + if len(tables) != 1: + logger.error(f"*** Not only a single table in {fname}.") + continue + last_updated_elem = next( + filter( + lambda e: isinstance(e, Tag) and e.name == "p", + soup.find(id="content").next_siblings, + ) + ) + last_updated_text = str(last_updated_elem.string).strip() + last_updated = datetime.strptime( + last_updated_text, "Last Updated: %m/%d/%Y" + ).date() + table = tables[0].find("tbody") + lines = table.find_all("tr") + if snapshot_date <= datetime(2020, 10, 28): + # NIST had a different format of the MIP table before this date, handle it. + entries = set() + for tr in lines: + tds = tr.find_all("td") + status = None + if "mip-highlight" in tds[-1]["class"]: + status = MIPStatus.FINALIZATION + elif "mip-highlight" in tds[-2]["class"]: + status = MIPStatus.COORDINATION + elif "mip-highlight" in tds[-3]["class"]: + status = MIPStatus.REVIEW_PENDING + elif "mip-highlight" in tds[-4]["class"]: + status = MIPStatus.IN_REVIEW + entries.add( + MIPEntry( + str(tds[0].string), + str(tds[1].string), + str(tds[2].string), + status, + ) + ) + else: + entries = { + MIPEntry( + str(line[0].string), + str(line[1].string), + str(line[2].string), + MIPStatus(str(line[3].string)), + ) + for line in map(lambda tr: tr.find_all("td"), lines) + } + snapshots.append(MIPSnapshot(entries, snapshot_date, last_updated)) + return cls(snapshots) + + def to_dict(self): + return { + "snapshots": list(self.snapshots) + } + + @classmethod + def from_dict(cls, dct: Mapping) -> "MIPDataset": + return cls( + dct["snapshots"] + ) + + def to_json(self, json_path: Union[str, Path]): + with open(json_path, 'w') as handle: + json.dump(self, handle, indent=4, cls=CustomJSONEncoder) + + @classmethod + def from_json(cls, json_path: Union[str, Path]): + with Path(json_path).open('r') as handle: + return json.load(handle, cls=CustomJSONDecoder) \ No newline at end of file diff --git a/sec_certs/iut.py b/sec_certs/iut.py deleted file mode 100644 index 1100eacd..00000000 --- a/sec_certs/iut.py +++ /dev/null @@ -1,128 +0,0 @@ -import json -import logging - -from dataclasses import dataclass -from typing import List, Set, Mapping, Union -from datetime import datetime, date -from pathlib import Path - -from tqdm import tqdm -from bs4 import BeautifulSoup, Tag -from sec_certs.serialization import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder - - -@dataclass(frozen=True) -class IUTEntry(ComplexSerializableType): - module_name: str - vendor_name: str - standard: str - iut_date: date - - def to_dict(self): - return {**self.__dict__, "iut_date": self.iut_date.isoformat()} - - @classmethod - def from_dict(cls, dct: Mapping) -> "IUTEntry": - return cls( - dct["module_name"], - dct["vendor_name"], - dct["standard"], - date.fromisoformat(dct["iut_date"]), - ) - - -@dataclass -class IUTSnapshot(ComplexSerializableType): - entries: Set[IUTEntry] - timestamp: datetime - last_updated: date - - def to_dict(self): - return { - "entries": list(self.entries), - "timestamp": self.timestamp.isoformat(), - "last_updated": self.last_updated.isoformat(), - } - - @classmethod - def from_dict(cls, dct: Mapping) -> "IUTSnapshot": - print(type(dct), type(dct["entries"])) - return cls( - set(dct["entries"]), - datetime.fromisoformat(dct["timestamp"]), - date.fromisoformat(dct["last_updated"]), - ) - - -@dataclass -class IUTDataset(ComplexSerializableType): - snapshots: List[IUTSnapshot] - - def __iter__(self): - yield from self.snapshots - - def __getitem__(self, item: int) -> IUTSnapshot: - return self.snapshots.__getitem__(item) - - def __len__(self) -> int: - return len(self.snapshots) - - @classmethod - def from_dump(cls, dump_path: Union[str, Path]) -> "IUTDataset": - directory = Path(dump_path) - snapshots = [] - fnames = list(directory.glob("*")) - for fname in tqdm(sorted(fnames), total=len(fnames)): - snapshot_date = to_utc( - datetime.fromisoformat(fname.name[len("fips_iut_") : -len(".html")]) - ) - with open(fname) as f: - soup = BeautifulSoup(f, "html.parser") - tables = soup.find_all("table") - if len(tables) != 1: - logger.error(f"*** Not only a single table in {fname}.") - continue - last_updated_elem = next( - filter( - lambda e: isinstance(e, Tag) and e.name == "p", - soup.find(id="content").next_siblings, - ) - ) - last_updated_text = str(last_updated_elem.string).strip() - last_updated = datetime.strptime( - last_updated_text, "Last Updated: %m/%d/%Y" - ).date() - table = tables[0].find("tbody") - lines = table.find_all("tr") - entries = { - IUTEntry( - str(line[0].string), - str(line[1].string), - str(line[2].string), - datetime.strptime(str(line[3].string), "%m/%d/%Y").date(), - ) - for line in map(lambda tr: tr.find_all("td"), lines) - } - snapshots.append(IUTSnapshot(entries, snapshot_date, last_updated)) - return cls(snapshots) - - - def to_dict(self): - return { - "snapshots": list(self.snapshots) - } - - @classmethod - def from_dict(cls, dct: Mapping) -> "IUTDataset": - return cls( - dct["snapshots"] - ) - - def to_json(self, json_path: Union[str, Path]): - with open(json_path, 'w') as handle: - json.dump(self, handle, indent=4, cls=CustomJSONEncoder) - - @classmethod - def from_json(cls, json_path: Union[str, Path]) -> "IUTDataset": - with Path(json_path).open('r') as handle: - return json.load(handle, cls=CustomJSONDecoder) diff --git a/sec_certs/mip.py b/sec_certs/mip.py deleted file mode 100644 index ec014b30..00000000 --- a/sec_certs/mip.py +++ /dev/null @@ -1,161 +0,0 @@ -import json -import logging - -from dataclasses import dataclass -from enum import Enum -from typing import List, Set, Mapping, Union -from datetime import datetime, date -from pathlib import Path - -from tqdm import tqdm -from bs4 import BeautifulSoup, Tag -from sec_certs.serialization import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder - - -logger = logging.getLogger(__name__) - - -class MIPStatus(Enum): - IN_REVIEW = "In Review" - REVIEW_PENDING = "Review Pending" - COORDINATION = "Coordination" - FINALIZATION = "Finalization" - - -@dataclass(frozen=True) -class MIPEntry(ComplexSerializableType): - module_name: str - vendor_name: str - standard: str - status: MIPStatus - - def to_dict(self): - return {**self.__dict__, "status": self.status.value} - - @classmethod - def from_dict(cls, dct: Mapping) -> "MIPEntry": - return cls( - dct["module_name"], - dct["vendor_name"], - dct["standard"], - MIPStatus(dct["status"]), - ) - - -@dataclass -class MIPSnapshot(ComplexSerializableType): - entries: Set[MIPEntry] - timestamp: datetime - last_updated: date - - def to_dict(self): - return { - "entries": list(self.entries), - "timestamp": self.timestamp.isoformat(), - "last_updated": self.last_updated.isoformat(), - } - - @classmethod - def from_dict(cls, dct: Mapping) -> "MIPSnapshot": - return cls( - set(dct["entries"]), - datetime.fromisoformat(dct["timestamp"]), - date.fromisoformat(dct["last_updated"]), - ) - - -@dataclass -class MIPDataset(ComplexSerializableType): - snapshots: List[MIPSnapshot] - - def __iter__(self): - yield from self.snapshots - - def __getitem__(self, item: int) -> MIPSnapshot: - return self.snapshots.__getitem__(item) - - def __len__(self) -> int: - return len(self.snapshots) - - @classmethod - def from_dump(cls, dump_path: Union[str, Path]) -> "MIPDataset": - directory = Path(dump_path) - snapshots = [] - fnames = list(directory.glob("*")) - for fname in tqdm(sorted(fnames), total=len(fnames)): - snapshot_date = to_utc( - datetime.fromisoformat(fname.name[len("fips_mip_") : -len(".html")]) - ) - with open(fname) as f: - soup = BeautifulSoup(f, "html.parser") - tables = soup.find_all("table") - if len(tables) != 1: - logger.error(f"*** Not only a single table in {fname}.") - continue - last_updated_elem = next( - filter( - lambda e: isinstance(e, Tag) and e.name == "p", - soup.find(id="content").next_siblings, - ) - ) - last_updated_text = str(last_updated_elem.string).strip() - last_updated = datetime.strptime( - last_updated_text, "Last Updated: %m/%d/%Y" - ).date() - table = tables[0].find("tbody") - lines = table.find_all("tr") - if snapshot_date <= datetime(2020, 10, 28): - # NIST had a different format of the MIP table before this date, handle it. - entries = set() - for tr in lines: - tds = tr.find_all("td") - status = None - if "mip-highlight" in tds[-1]["class"]: - status = MIPStatus.FINALIZATION - elif "mip-highlight" in tds[-2]["class"]: - status = MIPStatus.COORDINATION - elif "mip-highlight" in tds[-3]["class"]: - status = MIPStatus.REVIEW_PENDING - elif "mip-highlight" in tds[-4]["class"]: - status = MIPStatus.IN_REVIEW - entries.add( - MIPEntry( - str(tds[0].string), - str(tds[1].string), - str(tds[2].string), - status, - ) - ) - else: - entries = { - MIPEntry( - str(line[0].string), - str(line[1].string), - str(line[2].string), - MIPStatus(str(line[3].string)), - ) - for line in map(lambda tr: tr.find_all("td"), lines) - } - snapshots.append(MIPSnapshot(entries, snapshot_date, last_updated)) - return cls(snapshots) - - - def to_dict(self): - return { - "snapshots": list(self.snapshots) - } - - @classmethod - def from_dict(cls, dct: Mapping) -> "MIPDataset": - return cls( - dct["snapshots"] - ) - - def to_json(self, json_path: Union[str, Path]): - with open(json_path, 'w') as handle: - json.dump(self, handle, indent=4, cls=CustomJSONEncoder) - - @classmethod - def from_json(cls, json_path: Union[str, Path]): - with Path(json_path).open('r') as handle: - return json.load(handle, cls=CustomJSONDecoder) diff --git a/sec_certs/sample/fips_iut.py b/sec_certs/sample/fips_iut.py new file mode 100644 index 00000000..69bfa02c --- /dev/null +++ b/sec_certs/sample/fips_iut.py @@ -0,0 +1,47 @@ +from dataclasses import dataclass +from datetime import date, datetime +from typing import Mapping, Set + +from sec_certs.serialization.json import ComplexSerializableType + + +@dataclass(frozen=True) +class IUTEntry(ComplexSerializableType): + module_name: str + vendor_name: str + standard: str + iut_date: date + + def to_dict(self): + return {**self.__dict__, "iut_date": self.iut_date.isoformat()} + + @classmethod + def from_dict(cls, dct: Mapping) -> "IUTEntry": + return cls( + dct["module_name"], + dct["vendor_name"], + dct["standard"], + date.fromisoformat(dct["iut_date"]), + ) + + +@dataclass +class IUTSnapshot(ComplexSerializableType): + entries: Set[IUTEntry] + timestamp: datetime + last_updated: date + + def to_dict(self): + return { + "entries": list(self.entries), + "timestamp": self.timestamp.isoformat(), + "last_updated": self.last_updated.isoformat(), + } + + @classmethod + def from_dict(cls, dct: Mapping) -> "IUTSnapshot": + return cls( + set(dct["entries"]), + datetime.fromisoformat(dct["timestamp"]), + date.fromisoformat(dct["last_updated"]), + ) \ No newline at end of file diff --git a/sec_certs/sample/fips_mip.py b/sec_certs/sample/fips_mip.py new file mode 100644 index 00000000..a1441ddd --- /dev/null +++ b/sec_certs/sample/fips_mip.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass +from datetime import datetime, date +from enum import Enum +from typing import Mapping, Set + +from sec_certs.serialization.json import ComplexSerializableType + + +class MIPStatus(Enum): + IN_REVIEW = "In Review" + REVIEW_PENDING = "Review Pending" + COORDINATION = "Coordination" + FINALIZATION = "Finalization" + + +@dataclass(frozen=True) +class MIPEntry(ComplexSerializableType): + module_name: str + vendor_name: str + standard: str + status: MIPStatus + + def to_dict(self): + return {**self.__dict__, "status": self.status.value} + + @classmethod + def from_dict(cls, dct: Mapping) -> "MIPEntry": + return cls( + dct["module_name"], + dct["vendor_name"], + dct["standard"], + MIPStatus(dct["status"]), + ) + + +@dataclass +class MIPSnapshot(ComplexSerializableType): + entries: Set[MIPEntry] + timestamp: datetime + last_updated: date + + def to_dict(self): + return { + "entries": list(self.entries), + "timestamp": self.timestamp.isoformat(), + "last_updated": self.last_updated.isoformat(), + } + + @classmethod + def from_dict(cls, dct: Mapping) -> "MIPSnapshot": + return cls( + set(dct["entries"]), + datetime.fromisoformat(dct["timestamp"]), + date.fromisoformat(dct["last_updated"]), + ) -- cgit v1.3.1 From b5f91e04576f1c31ad93986d4578aa67f288ddea Mon Sep 17 00:00:00 2001 From: J08nY Date: Thu, 23 Dec 2021 19:58:24 +0100 Subject: Fix IUT/MIP demos. --- examples/fips_iut_demo.py | 3 ++- examples/fips_mip_demo.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'examples') diff --git a/examples/fips_iut_demo.py b/examples/fips_iut_demo.py index aaeaa103..f19f16e8 100755 --- a/examples/fips_iut_demo.py +++ b/examples/fips_iut_demo.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import click + from sec_certs.dataset.fips_iut import IUTDataset @@ -21,7 +22,7 @@ def main(directory, output): in_process.py fips-iut d output.json to obtain the parsed output in `output.json`. """ - dataset = IUTDataset.from_dump(directory) + dataset = IUTDataset.from_dumps(directory) dataset.to_json(output) diff --git a/examples/fips_mip_demo.py b/examples/fips_mip_demo.py index bff6f3a1..643ee993 100755 --- a/examples/fips_mip_demo.py +++ b/examples/fips_mip_demo.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import click + from sec_certs.dataset.fips_mip import MIPDataset @@ -21,7 +22,7 @@ def main(directory, output): in_process.py fips-mip d output.json to obtain the parsed output in `output.json`. """ - dataset = MIPDataset.from_dump(directory) + dataset = MIPDataset.from_dumps(directory) dataset.to_json(output) -- cgit v1.3.1