diff options
| author | J08nY | 2023-02-09 21:14:42 +0100 |
|---|---|---|
| committer | J08nY | 2023-02-09 21:14:42 +0100 |
| commit | 997143c1335bea2a86da4284c57d3dcd9fe20f47 (patch) | |
| tree | f3a5db15e57cad1bfb76fdb86b49c44a861e2d27 /src | |
| parent | a13e085b06cb2772ec812e4bd364a7fe938c806f (diff) | |
| download | sec-certs-997143c1335bea2a86da4284c57d3dcd9fe20f47.tar.gz sec-certs-997143c1335bea2a86da4284c57d3dcd9fe20f47.tar.zst sec-certs-997143c1335bea2a86da4284c57d3dcd9fe20f47.zip | |
Fix some failing CC scheme downloads.
Diffstat (limited to 'src')
| -rw-r--r-- | src/sec_certs/constants.py | 5 | ||||
| -rw-r--r-- | src/sec_certs/dataset/cc.py | 23 |
2 files changed, 16 insertions, 12 deletions
diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py index f71bb3c3..684662d0 100644 --- a/src/sec_certs/constants.py +++ b/src/sec_certs/constants.py @@ -74,7 +74,10 @@ CC_CANADA_INEVAL_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteri CC_ANSSI_BASE_URL = "https://www.ssi.gouv.fr" CC_ANSSI_CERTIFIED_URL = CC_ANSSI_BASE_URL + "/en/products/certified-products/" CC_BSI_BASE_URL = "https://www.bsi.bund.de/" -CC_BSI_CERTIFIED_URL = CC_BSI_BASE_URL + "EN2021/Topics/Certification/certified_products/certified_products_node.html" +CC_BSI_CERTIFIED_URL = ( + CC_BSI_BASE_URL + + "EN/Themen/Unternehmen-und-Organisationen/Standards-und-Zertifizierung/Zertifizierung-und-Anerkennung/Listen/Zertifizierte-Produkte-nach-CC/zertifizierte-produkte-nach-cc_node.html" +) CC_INDIA_CERTIFIED_URL = "https://www.commoncriteria-india.gov.in/product-certified" CC_INDIA_ARCHIVED_URL = "https://www.commoncriteria-india.gov.in/archived-prod-cer" CC_ITALY_BASE_URL = "https://www.ocsi.gov.it" diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py index bb806d18..cf93f651 100644 --- a/src/sec_certs/dataset/cc.py +++ b/src/sec_certs/dataset/cc.py @@ -894,7 +894,7 @@ class CCSchemeDataset: @staticmethod def _download_page(url, session=None): conn = session if session else requests - resp = conn.get(url, headers={"User-Agent": "seccerts.org"}) + resp = conn.get(url, headers={"User-Agent": "seccerts.org"}, verify=False) if resp.status_code != requests.codes.ok: raise ValueError(f"Unable to download: status={resp.status_code}") return BeautifulSoup(resp.content, "html5lib") @@ -1033,14 +1033,14 @@ class CCSchemeDataset: pager = soup.find("ul", class_="pager") for li in pager.find_all("li"): try: - new_page = int(li.text) + new_page = int(li.text) - 1 except Exception: continue if new_page not in seen_pages: pages.add(new_page) # Parse table - tbody = soup.find("div", class_="content").find("table").find("tbody") + tbody = soup.find("div", class_="view-content").find("table").find("tbody") for tr in tbody.find_all("tr"): tds = tr.find_all("td") if not tds: @@ -1077,16 +1077,17 @@ class CCSchemeDataset: # Update pages pager = soup.find("ul", class_="pager") - for li in pager.find_all("li"): - try: - new_page = int(li.text) - except Exception: - continue - if new_page not in seen_pages: - pages.add(new_page) + if pager: + for li in pager.find_all("li"): + try: + new_page = int(li.text) - 1 + except Exception: + continue + if new_page not in seen_pages: + pages.add(new_page) # Parse table - tbody = soup.find("div", class_="content").find("table").find("tbody") + tbody = soup.find("div", class_="view-content").find("table").find("tbody") for tr in tbody.find_all("tr"): tds = tr.find_all("td") if not tds: |