diff options
| author | J08nY | 2024-01-02 17:13:08 +0100 |
|---|---|---|
| committer | J08nY | 2024-01-02 17:13:08 +0100 |
| commit | 7ad0eed7510a08740efd195bf4ee2f89a719dcc6 (patch) | |
| tree | e9303fdb202f6bd9027c761b5900eb2cafead42c /src | |
| parent | 70c7c4f88a2da5f027558951eeb9ab5f85aa03b0 (diff) | |
| download | sec-certs-7ad0eed7510a08740efd195bf4ee2f89a719dcc6.tar.gz sec-certs-7ad0eed7510a08740efd195bf4ee2f89a719dcc6.tar.zst sec-certs-7ad0eed7510a08740efd195bf4ee2f89a719dcc6.zip | |
Fix CC CSV and HTML parsing.
Diffstat (limited to 'src')
| -rw-r--r-- | src/sec_certs/dataset/cc.py | 7 | ||||
| -rw-r--r-- | src/sec_certs/sample/cc.py | 20 |
2 files changed, 18 insertions, 9 deletions
diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py index 04db4d7c..4eee93b6 100644 --- a/src/sec_certs/dataset/cc.py +++ b/src/sec_certs/dataset/cc.py @@ -178,7 +178,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable BASE_URL: ClassVar[str] = "https://www.commoncriteriaportal.org" HTML_PRODUCTS_URL = { - "cc_products_active.html": BASE_URL + "/products/", + "cc_products_active.html": BASE_URL + "/products/index.cfm", "cc_products_archived.html": BASE_URL + "/products/index.cfm?archived=1", } HTML_LABS_URL = {"cc_labs.html": BASE_URL + "/labs"} @@ -362,7 +362,10 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable ] # TODO: Now skipping bad lines, smarter heuristics to be built for dumb files - df = pd.read_csv(file, engine="python", encoding="windows-1252", on_bad_lines="skip") + try: + df = pd.read_csv(file, engine="python", encoding="utf-8", on_bad_lines="skip") + except UnicodeDecodeError: + df = pd.read_csv(file, engine="python", encoding="windows-1252", on_bad_lines="skip") df = df.rename(columns=dict(zip(list(df.columns), csv_header))) df["is_maintenance"] = ~df.maintenance_title.isnull() diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py index 32c875c8..4cd1324d 100644 --- a/src/sec_certs/sample/cc.py +++ b/src/sec_certs/sample/cc.py @@ -486,8 +486,8 @@ class CCCertificate( security_level: str | set[str], not_valid_before: date | None, not_valid_after: date | None, - report_link: str, - st_link: str, + report_link: str | None, + st_link: str | None, cert_link: str | None, manufacturer_web: str | None, protection_profiles: set[ProtectionProfile] | None, @@ -694,13 +694,19 @@ class CCCertificate( return extracted_date @staticmethod - def _html_row_get_report_st_links(cell: Tag) -> tuple[str, str]: + def _html_row_get_report_st_links(cell: Tag) -> tuple[str | None, str | None]: links = cell.find_all("a") - assert links[1].get("title").startswith("Certification Report") - assert links[2].get("title").startswith("Security Target") - report_link = CCCertificate.cc_url + links[1].get("href") - security_target_link = CCCertificate.cc_url + links[2].get("href") + report_link: str | None = None + security_target_link: str | None = None + for link in links: + title = link.get("title") + if not title: + continue + if title.startswith("Certification Report"): + report_link = CCCertificate.cc_url + link.get("href") + elif title.startswith("Security Target"): + security_target_link = CCCertificate.cc_url + link.get("href") return report_link, security_target_link |