Use urljoin and make URLs absolute in CC scheme dataset.

3 files changed, 199 insertions, 109 deletions

diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py
index 0222b2b9..771220cd 100644
--- a/src/sec_certs/constants.py
+++ b/src/sec_certs/constants.py
@@ -66,8 +66,9 @@ GARBAGE_EVERY_SECOND_CHAR_THRESHOLD = 15
 GARBAGE_ALPHA_CHARS_THRESHOLD = 0.5
 
 CC_AUSTRALIA_BASE_URL = "https://www.cyber.gov.au"
-CC_AUSTRALIA_CERTIFIED_URL = (
-    CC_AUSTRALIA_BASE_URL + "/acsc/view-all-content/programs/australian-information-security-evaluation-program"
+CC_AUSTRALIA_INEVAL_URL = (
+    CC_AUSTRALIA_BASE_URL
+    + "/resources-business-and-government/assessment-and-evaluation-programs/australian-information-security-evaluation-program-aisep"
 )
 CC_CANADA_CERTIFIED_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/certified-products"
 CC_CANADA_INEVAL_URL = "https://www.cyber.gc.ca/en/tools-services/common-criteria/products-evaluation"
@@ -78,8 +79,9 @@ CC_BSI_CERTIFIED_URL = (
     CC_BSI_BASE_URL
     + "EN/Themen/Unternehmen-und-Organisationen/Standards-und-Zertifizierung/Zertifizierung-und-Anerkennung/Listen/Zertifizierte-Produkte-nach-CC/zertifizierte-produkte-nach-cc_node.html"
 )
-CC_INDIA_CERTIFIED_URL = "https://www.commoncriteria-india.gov.in/Products-Certified"
-CC_INDIA_ARCHIVED_URL = "https://www.commoncriteria-india.gov.in/Products-Archived"
+CC_INDIA_BASE_URL = "https://www.commoncriteria-india.gov.in"
+CC_INDIA_CERTIFIED_URL = CC_INDIA_BASE_URL + "/Products-Certified"
+CC_INDIA_ARCHIVED_URL = CC_INDIA_BASE_URL + "/Products-Archived"
 CC_ITALY_BASE_URL = "https://www.ocsi.gov.it"
 CC_ITALY_CERTIFIED_URL = CC_ITALY_BASE_URL + "/index.php/elenchi-certificazioni/prodotti-certificati.html"
 CC_ITALY_INEVAL_URL = CC_ITALY_BASE_URL + "/index.php/elenchi-certificazioni/in-corso-di-valutazione.html"
@@ -125,6 +127,7 @@ CC_SWEDEN_INEVAL_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/p
 CC_SWEDEN_ARCHIVED_URL = CC_SWEDEN_BASE_URL + "/verksamhet/ovrig-verksamhet/csec/arkiverade-certifikat-aldre-an-5-ar/"
 CC_TURKEY_ARCHIVED_URL = "https://statik.tse.org.tr/upload/tr/dosya/icerikyonetimi/3300/03112021143434-2.pdf"
 CC_USA_BASE_URL = "https://www.niap-ccevs.org"
+CC_USA_PRODUCT_URL = CC_USA_BASE_URL + "/Product"
 CC_USA_CERTIFIED_URL = CC_USA_BASE_URL + "/Product/PCL.cfm"
 CC_USA_INEVAL_URL = CC_USA_BASE_URL + "/Product/PINE.cfm"
 CC_USA_ARCHIVED_URL = CC_USA_BASE_URL + "/Product/Archived.cfm"
diff --git a/src/sec_certs/dataset/cc_scheme.py b/src/sec_certs/dataset/cc_scheme.py
index 84e3a24f..01781e15 100644
--- a/src/sec_certs/dataset/cc_scheme.py
+++ b/src/sec_certs/dataset/cc_scheme.py
@@ -1,5 +1,6 @@
 import tempfile
 from pathlib import Path
+from urllib.parse import urljoin
 
 import requests
 import tabula
@@ -11,17 +12,16 @@ from sec_certs.utils.sanitization import sanitize_navigable_string as sns
 
 class CCSchemeDataset:
     @staticmethod
-    def _download_page(url, session=None):
+    def _get_page(url, session=None):
         conn = session if session else requests
         resp = conn.get(url, headers={"User-Agent": "seccerts.org"}, verify=False)
-        if resp.status_code != requests.codes.ok:
-            raise ValueError(f"Unable to download: status={resp.status_code}")
+        resp.raise_for_status()
         return BeautifulSoup(resp.content, "html5lib")
 
     @staticmethod
     def get_australia_in_evaluation():
         # TODO: Information could be expanded by following url.
-        soup = CCSchemeDataset._download_page(constants.CC_AUSTRALIA_CERTIFIED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_AUSTRALIA_INEVAL_URL)
         header = soup.find("h2", text="Products in evaluation")
         table = header.find_next_sibling("table")
         results = []
@@ -29,18 +29,20 @@ class CCSchemeDataset:
             tds = tr.find_all("td")
             if not tds:
                 continue
+            cert_url = urljoin(constants.CC_AUSTRALIA_BASE_URL, tds[1].find("a")["href"])
             cert = {
                 "vendor": sns(tds[0].text),
                 "product": sns(tds[1].text),
-                "url": constants.CC_AUSTRALIA_BASE_URL + tds[1].find("a")["href"],
+                "url": cert_url,
                 "level": sns(tds[2].text),
             }
+            print(cert)
             results.append(cert)
         return results
 
     @staticmethod
     def get_canada_certified():
-        soup = CCSchemeDataset._download_page(constants.CC_CANADA_CERTIFIED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_CANADA_CERTIFIED_URL)
         tbody = soup.find("table").find("tbody")
         results = []
         for tr in tbody.find_all("tr"):
@@ -58,7 +60,7 @@ class CCSchemeDataset:
 
     @staticmethod
     def get_canada_in_evaluation():
-        soup = CCSchemeDataset._download_page(constants.CC_CANADA_INEVAL_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_CANADA_INEVAL_URL)
         tbody = soup.find("table").find("tbody")
         results = []
         for tr in tbody.find_all("tr"):
@@ -77,14 +79,14 @@ class CCSchemeDataset:
     @staticmethod
     def get_france_certified():
         # TODO: Information could be expanded by following product link.
-        base_soup = CCSchemeDataset._download_page(constants.CC_ANSSI_CERTIFIED_URL)
+        base_soup = CCSchemeDataset._get_page(constants.CC_ANSSI_CERTIFIED_URL)
         category_nav = base_soup.find("ul", class_="nav-categories")
         results = []
         for li in category_nav.find_all("li"):
             a = li.find("a")
             url = a["href"]
             category_name = sns(a.text)
-            soup = CCSchemeDataset._download_page(constants.CC_ANSSI_BASE_URL + url)
+            soup = CCSchemeDataset._get_page(urljoin(constants.CC_ANSSI_BASE_URL, url))
             table = soup.find("table", class_="produits-liste cc")
             if not table:
                 continue
@@ -100,7 +102,7 @@ class CCSchemeDataset:
                     "id": sns(tds[3].text),
                     "certification_date": sns(tds[4].text),
                     "category": category_name,
-                    "url": constants.CC_ANSSI_BASE_URL + tds[0].find("a")["href"],
+                    "url": urljoin(constants.CC_ANSSI_BASE_URL, tds[0].find("a")["href"]),
                 }
                 results.append(cert)
         return results
@@ -108,14 +110,14 @@ class CCSchemeDataset:
     @staticmethod
     def get_germany_certified():
         # TODO: Information could be expanded by following url.
-        base_soup = CCSchemeDataset._download_page(constants.CC_BSI_CERTIFIED_URL)
+        base_soup = CCSchemeDataset._get_page(constants.CC_BSI_CERTIFIED_URL)
         category_nav = base_soup.find("ul", class_="no-bullet row")
         results = []
         for li in category_nav.find_all("li"):
             a = li.find("a")
             url = a["href"]
             category_name = sns(a.text)
-            soup = CCSchemeDataset._download_page(constants.CC_BSI_BASE_URL + url)
+            soup = CCSchemeDataset._get_page(urljoin(constants.CC_BSI_BASE_URL, url))
             content = soup.find("div", class_="content").find("div", class_="column")
             for table in content.find_all("table"):
                 tbody = table.find("tbody")
@@ -130,7 +132,7 @@ class CCSchemeDataset:
                         "vendor": sns(tds[2].text),
                         "certification_date": sns(tds[3].text),
                         "category": category_name,
-                        "url": constants.CC_BSI_BASE_URL + tds[0].find("a")["href"],
+                        "url": urljoin(constants.CC_BSI_BASE_URL, tds[0].find("a")["href"]),
                     }
                     if header is not None:
                         cert["subcategory"] = sns(header.text)
@@ -138,6 +140,10 @@ class CCSchemeDataset:
         return results
 
     @staticmethod
+    def _fix_india_link(link):
+        return link.replace("/index.php", "")
+
+    @staticmethod
     def get_india_certified():
         pages = {0}
         seen_pages = set()
@@ -146,7 +152,7 @@ class CCSchemeDataset:
             page = pages.pop()
             seen_pages.add(page)
             url = constants.CC_INDIA_CERTIFIED_URL + f"?page={page}"
-            soup = CCSchemeDataset._download_page(url)
+            soup = CCSchemeDataset._get_page(url)
 
             # Update pages
             pager = soup.find("ul", class_="pager__items")
@@ -174,11 +180,15 @@ class CCSchemeDataset:
                     "developer": sns(tds[3].text),
                     "level": sns(tds[4].text),
                     "issuance_date": sns(tds[5].text),
-                    "report_link": report_a["href"],
+                    "report_link": urljoin(
+                        constants.CC_INDIA_BASE_URL, CCSchemeDataset._fix_india_link(report_a["href"])
+                    ),
                     "report_name": sns(report_a.text),
-                    "target_link": target_a["href"],
+                    "target_link": urljoin(
+                        constants.CC_INDIA_BASE_URL, CCSchemeDataset._fix_india_link(target_a["href"])
+                    ),
                     "target_name": sns(target_a.text),
-                    "cert_link": cert_a["href"],
+                    "cert_link": urljoin(constants.CC_INDIA_BASE_URL, CCSchemeDataset._fix_india_link(cert_a["href"])),
                     "cert_name": sns(cert_a.text),
                 }
                 results.append(cert)
@@ -193,7 +203,7 @@ class CCSchemeDataset:
             page = pages.pop()
             seen_pages.add(page)
             url = constants.CC_INDIA_ARCHIVED_URL + f"?page={page}"
-            soup = CCSchemeDataset._download_page(url)
+            soup = CCSchemeDataset._get_page(url)
 
             # Update pages
             pager = soup.find("ul", class_="pager__items")
@@ -221,21 +231,25 @@ class CCSchemeDataset:
                     "sponsor": sns(tds[2].text),
                     "developer": sns(tds[3].text),
                     "level": sns(tds[4].text),
-                    "target_link": target_a["href"],
+                    "target_link": urljoin(
+                        constants.CC_INDIA_BASE_URL, CCSchemeDataset._fix_india_link(target_a["href"])
+                    ),
                     "target_name": sns(target_a.text),
-                    "cert_link": cert_a["href"],
+                    "cert_link": urljoin(constants.CC_INDIA_BASE_URL, CCSchemeDataset._fix_india_link(cert_a["href"])),
                     "cert_name": sns(cert_a.text),
                     "certification_date": sns(tds[8].text),
                 }
                 if report_a:
-                    cert["report_link"] = report_a["href"]
+                    cert["report_link"] = urljoin(
+                        constants.CC_INDIA_BASE_URL, CCSchemeDataset._fix_india_link(report_a["href"])
+                    )
                     cert["report_name"] = sns(report_a.text)
                 results.append(cert)
         return results
 
     @staticmethod
     def get_italy_certified():  # noqa: C901
-        soup = CCSchemeDataset._download_page(constants.CC_ITALY_CERTIFIED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_ITALY_CERTIFIED_URL)
         div = soup.find("div", class_="certificati")
         results = []
         for cert_div in div.find_all("div", recursive=False):
@@ -258,21 +272,21 @@ class CCSchemeDataset:
                 elif "Data revisione" in p_name:
                     cert["revision_date"] = p_data
                 elif "Rapporto di Certificazione" in p_name and p_link:
-                    cert["report_link_it"] = constants.CC_ITALY_BASE_URL + p_link["href"]
+                    cert["report_link_it"] = urljoin(constants.CC_ITALY_BASE_URL, p_link["href"])
                 elif "Certification Report" in p_name and p_link:
-                    cert["report_link_en"] = constants.CC_ITALY_BASE_URL + p_link["href"]
+                    cert["report_link_en"] = urljoin(constants.CC_ITALY_BASE_URL, p_link["href"])
                 elif "Traguardo di Sicurezza" in p_name and p_link:
-                    cert["target_link"] = constants.CC_ITALY_BASE_URL + p_link["href"]
+                    cert["target_link"] = urljoin(constants.CC_ITALY_BASE_URL, p_link["href"])
                 elif "Nota su" in p_name and p_link:
-                    cert["vulnerability_note_link"] = constants.CC_ITALY_BASE_URL + p_link["href"]
+                    cert["vulnerability_note_link"] = urljoin(constants.CC_ITALY_BASE_URL, p_link["href"])
                 elif "Nota di chiarimento" in p_name and p_link:
-                    cert["clarification_note_link"] = constants.CC_ITALY_BASE_URL + p_link["href"]
+                    cert["clarification_note_link"] = urljoin(constants.CC_ITALY_BASE_URL, p_link["href"])
             results.append(cert)
         return results
 
     @staticmethod
     def get_italy_in_evaluation():
-        soup = CCSchemeDataset._download_page(constants.CC_ITALY_INEVAL_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_ITALY_INEVAL_URL)
         div = soup.find("div", class_="valutazioni")
         results = []
         for cert_div in div.find_all("div", recursive=False):
@@ -297,7 +311,7 @@ class CCSchemeDataset:
     @staticmethod
     def _get_japan(url):
         # TODO: Information could be expanded by following toe link.
-        soup = CCSchemeDataset._download_page(url)
+        soup = CCSchemeDataset._get_page(url)
         table = soup.find("table", class_="cert-table")
         results = []
         trs = list(table.find_all("tr"))
@@ -315,14 +329,14 @@ class CCSchemeDataset:
                 }
                 toe_a = tds[2].find("a")
                 if toe_a and "href" in toe_a.attrs:
-                    cert["toe_overseas_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"]
+                    cert["toe_overseas_link"] = urljoin(constants.CC_JAPAN_CERT_BASE_URL, "/" + toe_a["href"])
                 results.append(cert)
             if len(tds) == 1:
                 cert = results[-1]
                 cert["toe_japan_name"] = sns(tds[0].text)
                 toe_a = tds[0].find("a")
                 if toe_a and "href" in toe_a.attrs:
-                    cert["toe_japan_link"] = constants.CC_JAPAN_CERT_BASE_URL + "/" + toe_a["href"]
+                    cert["toe_japan_link"] = urljoin(constants.CC_JAPAN_CERT_BASE_URL, "/" + toe_a["href"])
         return results
 
     @staticmethod
@@ -338,7 +352,7 @@ class CCSchemeDataset:
     @staticmethod
     def get_japan_in_evaluation():
         # TODO: Information could be expanded by following toe link.
-        soup = CCSchemeDataset._download_page(constants.CC_JAPAN_INEVAL_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_JAPAN_INEVAL_URL)
         table = soup.find("table")
         results = []
         for tr in table.find_all("tr"):
@@ -349,7 +363,7 @@ class CCSchemeDataset:
             cert = {
                 "supplier": sns(tds[0].text),
                 "toe_name": sns(toe_a.text),
-                "toe_link": constants.CC_JAPAN_BASE_URL + "/" + toe_a["href"],
+                "toe_link": urljoin(constants.CC_JAPAN_BASE_URL, "/" + toe_a["href"]),
                 "claim": sns(tds[2].text),
             }
             results.append(cert)
@@ -357,7 +371,7 @@ class CCSchemeDataset:
 
     @staticmethod
     def get_malaysia_certified():
-        soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_CERTIFIED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_MALAYSIA_CERTIFIED_URL)
         sections = soup.find("div", attrs={"itemprop": "articleBody"}).find_all("section", class_="sppb-section")
         results = []
         for section in sections:
@@ -386,7 +400,7 @@ class CCSchemeDataset:
 
     @staticmethod
     def get_malaysia_in_evaluation():
-        soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_INEVAL_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_MALAYSIA_INEVAL_URL)
         main_div = soup.find("div", attrs={"itemprop": "articleBody"})
         table = main_div.find("table")
         results = []
@@ -406,7 +420,7 @@ class CCSchemeDataset:
 
     @staticmethod
     def get_netherlands_certified():
-        soup = CCSchemeDataset._download_page(constants.CC_NETHERLANDS_CERTIFIED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_NETHERLANDS_CERTIFIED_URL)
         main_div = soup.select("body > main > div > div > div > div:nth-child(2) > div.col-lg-9 > div:nth-child(3)")[0]
         rows = main_div.find_all("div", class_="row", recursive=False)
         modals = main_div.find_all("div", class_="modal", recursive=False)
@@ -428,19 +442,19 @@ class CCSchemeDataset:
                 elif "Assurancelevel" in th_text:
                     cert["level"] = sns(td.text)
                 elif "Certificate" in th_text:
-                    cert["cert_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"]
+                    cert["cert_link"] = urljoin(constants.CC_NETHERLANDS_BASE_URL, td.find("a")["href"])
                 elif "Certificationreport" in th_text:
-                    cert["report_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"]
+                    cert["report_link"] = urljoin(constants.CC_NETHERLANDS_BASE_URL, td.find("a")["href"])
                 elif "Securitytarget" in th_text:
-                    cert["target_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"]
+                    cert["target_link"] = urljoin(constants.CC_NETHERLANDS_BASE_URL, td.find("a")["href"])
                 elif "Maintenance report" in th_text:
-                    cert["maintenance_link"] = constants.CC_NETHERLANDS_BASE_URL + td.find("a")["href"]
+                    cert["maintenance_link"] = urljoin(constants.CC_NETHERLANDS_BASE_URL, td.find("a")["href"])
             results.append(cert)
         return results
 
     @staticmethod
     def get_netherlands_in_evaluation():
-        soup = CCSchemeDataset._download_page(constants.CC_NETHERLANDS_INEVAL_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_NETHERLANDS_INEVAL_URL)
         table = soup.find("table")
         results = []
         for tr in table.find_all("tr")[1:]:
@@ -458,7 +472,7 @@ class CCSchemeDataset:
     @staticmethod
     def _get_norway(url):
         # TODO: Information could be expanded by following product link.
-        soup = CCSchemeDataset._download_page(url)
+        soup = CCSchemeDataset._get_page(url)
         results = []
         for tr in soup.find_all("tr", class_="certified-product"):
             tds = tr.find_all("td")
@@ -487,7 +501,7 @@ class CCSchemeDataset:
         session.get(constants.CC_KOREA_EN_URL)
         # Get base page
         url = constants.CC_KOREA_CERTIFIED_URL + f"?product_class={product_class}"
-        soup = CCSchemeDataset._download_page(url, session=session)
+        soup = CCSchemeDataset._get_page(url, session=session)
         seen_pages = set()
         pages = {1}
         results = []
@@ -538,7 +552,7 @@ class CCSchemeDataset:
 
     @staticmethod
     def _get_singapore(url):
-        soup = CCSchemeDataset._download_page(url)
+        soup = CCSchemeDataset._get_page(url)
         page_id = str(soup.find("input", id="CurrentPageId").value)
         page = 1
         api_call = requests.post(
@@ -560,16 +574,16 @@ class CCSchemeDataset:
                     "level": obj["assuranceLevel"],
                     "product": obj["productName"],
                     "vendor": obj["productDeveloper"],
-                    "url": constants.CC_SINGAPORE_BASE_URL + obj["productUrl"],
+                    "url": urljoin(constants.CC_SINGAPORE_BASE_URL, obj["productUrl"]),
                     "certification_date": obj["dateOfIssuance"],
                     "expiration_date": obj["dateOfExpiry"],
                     "category": obj["productCategory"]["title"],
                     "cert_title": obj["certificate"]["title"],
-                    "cert_link": constants.CC_SINGAPORE_BASE_URL + obj["certificate"]["mediaUrl"],
+                    "cert_link": urljoin(constants.CC_SINGAPORE_BASE_URL, obj["certificate"]["mediaUrl"]),
                     "report_title": obj["certificationReport"]["title"],
-                    "report_link": constants.CC_SINGAPORE_BASE_URL + obj["certificationReport"]["mediaUrl"],
+                    "report_link": urljoin(constants.CC_SINGAPORE_BASE_URL, obj["certificationReport"]["mediaUrl"]),
                     "target_title": obj["securityTarget"]["title"],
-                    "target_link": constants.CC_SINGAPORE_BASE_URL + obj["securityTarget"]["mediaUrl"],
+                    "target_link": urljoin(constants.CC_SINGAPORE_BASE_URL, obj["securityTarget"]["mediaUrl"]),
                 }
                 results.append(cert)
             page += 1
@@ -592,7 +606,7 @@ class CCSchemeDataset:
 
     @staticmethod
     def get_singapore_in_evaluation():
-        soup = CCSchemeDataset._download_page(constants.CC_SINGAPORE_INEVAL_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_SINGAPORE_INEVAL_URL)
         blocks = soup.find_all("div", class_="sfContentBlock")
         for block in blocks:
             table = block.find("table")
@@ -617,14 +631,14 @@ class CCSchemeDataset:
 
     @staticmethod
     def get_spain_certified():
-        soup = CCSchemeDataset._download_page(constants.CC_SPAIN_CERTIFIED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_SPAIN_CERTIFIED_URL)
         tbody = soup.find("table", class_="djc_items_table").find("tbody")
         results = []
         for tr in tbody.find_all("tr", recursive=False):
             tds = tr.find_all("td")
             cert = {
                 "product": sns(tds[0].text),
-                "product_link": constants.CC_SPAIN_BASE_URL + tds[0].find("a")["href"],
+                "product_link": urljoin(constants.CC_SPAIN_BASE_URL, tds[0].find("a")["href"]),
                 "category": sns(tds[1].text),
                 "manufacturer": sns(tds[2].text),
                 "certification_date": sns(tds[3].find("td", class_="djc_value").text),
@@ -635,11 +649,11 @@ class CCSchemeDataset:
     @staticmethod
     def _get_sweden(url):
         # TODO: Information could be expanded by following product link.
-        soup = CCSchemeDataset._download_page(url)
+        soup = CCSchemeDataset._get_page(url)
         nav = soup.find("main").find("nav", class_="component-nav-box__list")
         results = []
         for link in nav.find_all("a"):
-            cert = {"product": sns(link.text), "product_link": constants.CC_SWEDEN_BASE_URL + link["href"]}
+            cert = {"product": sns(link.text), "product_link": urljoin(constants.CC_SWEDEN_BASE_URL, link["href"])}
             results.append(cert)
         return results
 
@@ -688,7 +702,7 @@ class CCSchemeDataset:
     def get_usa_certified():
         # TODO: Information could be expanded by following product link.
         # TODO: Information could be expanded by following the cc_claims (has links to protection profiles).
-        soup = CCSchemeDataset._download_page(constants.CC_USA_CERTIFIED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_USA_CERTIFIED_URL)
         tbody = soup.find("table", class_="tablesorter").find("tbody")
         results = []
         for tr in tbody.find_all("tr"):
@@ -702,7 +716,7 @@ class CCSchemeDataset:
             cert = {
                 "product": sns(product_link.text),
                 "vendor": sns(vendor_span.text),
-                "product_link": product_link["href"],
+                "product_link": urljoin(constants.CC_USA_PRODUCT_URL, product_link["href"]),
                 "id": sns(tds[1].text),
                 "cc_claim": sns(tds[2].text),
                 "cert_lab": sns(tds[3].text),
@@ -715,7 +729,7 @@ class CCSchemeDataset:
     @staticmethod
     def get_usa_in_evaluation():
         # TODO: Information could be expanded by following the cc_claims (has links to protection profiles).
-        soup = CCSchemeDataset._download_page(constants.CC_USA_INEVAL_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_USA_INEVAL_URL)
         tbody = soup.find("table", class_="tablesorter").find("tbody")
         results = []
         for tr in tbody.find_all("tr"):
@@ -741,7 +755,7 @@ class CCSchemeDataset:
     @staticmethod
     def get_usa_archived():
         # TODO: Information could be expanded by following the cc_claims (has links to protection profiles).
-        soup = CCSchemeDataset._download_page(constants.CC_USA_ARCHIVED_URL)
+        soup = CCSchemeDataset._get_page(constants.CC_USA_ARCHIVED_URL)
         tbody = soup.find("table", class_="tablesorter").find("tbody")
         results = []
         for tr in tbody.find_all("tr"):
diff --git a/tests/cc/test_cc_schemes.py b/tests/cc/test_cc_schemes.py
index 078a3190..230383fa 100644
--- a/tests/cc/test_cc_schemes.py
+++ b/tests/cc/test_cc_schemes.py
@@ -1,99 +1,172 @@
+from urllib.parse import urlparse
+
 import pytest
+from requests import RequestException
 
 from sec_certs.dataset import CCSchemeDataset
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+def absolute_urls(results):
+    for result in results:
+        for key, value in result.items():
+            if "url" in key or "link" in key:
+                parsed = urlparse(value)
+                assert bool(parsed.netloc)
+    return True
+
+
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_australia():
-    assert len(CCSchemeDataset.get_australia_in_evaluation()) != 0
+    ineval = CCSchemeDataset.get_australia_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_canada():
-    assert len(CCSchemeDataset.get_canada_certified()) != 0
-    assert len(CCSchemeDataset.get_canada_in_evaluation()) != 0
+    certified = CCSchemeDataset.get_canada_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    ineval = CCSchemeDataset.get_canada_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_anssi():
-    assert len(CCSchemeDataset.get_france_certified()) != 0
+    certified = CCSchemeDataset.get_france_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_bsi():
-    assert len(CCSchemeDataset.get_germany_certified()) != 0
+    certified = CCSchemeDataset.get_germany_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_india():
-    assert len(CCSchemeDataset.get_india_certified()) != 0
-    assert len(CCSchemeDataset.get_india_archived()) != 0
+    certified = CCSchemeDataset.get_india_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    archived = CCSchemeDataset.get_india_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_italy():
-    assert len(CCSchemeDataset.get_italy_certified()) != 0
-    assert len(CCSchemeDataset.get_italy_in_evaluation()) != 0
+    certified = CCSchemeDataset.get_italy_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    ineval = CCSchemeDataset.get_italy_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_japan():
-    assert len(CCSchemeDataset.get_japan_certified()) != 0
-    assert len(CCSchemeDataset.get_japan_archived()) != 0
-    assert len(CCSchemeDataset.get_japan_in_evaluation()) != 0
+    certified = CCSchemeDataset.get_japan_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    archived = CCSchemeDataset.get_japan_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
+    ineval = CCSchemeDataset.get_japan_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_malaysia():
-    assert len(CCSchemeDataset.get_malaysia_certified()) != 0
-    assert len(CCSchemeDataset.get_malaysia_in_evaluation()) != 0
+    certified = CCSchemeDataset.get_malaysia_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    ineval = CCSchemeDataset.get_malaysia_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_netherlands():
-    assert len(CCSchemeDataset.get_netherlands_certified()) != 0
-    assert len(CCSchemeDataset.get_netherlands_in_evaluation()) != 0
+    certified = CCSchemeDataset.get_netherlands_certified()
+    assert len(certified) != 0
+    # assert absolute_urls(certified)
+    ineval = CCSchemeDataset.get_netherlands_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_norway():
-    assert len(CCSchemeDataset.get_norway_certified()) != 0
-    assert len(CCSchemeDataset.get_norway_archived()) != 0
+    certified = CCSchemeDataset.get_norway_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    archived = CCSchemeDataset.get_norway_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_korea():
-    assert len(CCSchemeDataset.get_korea_certified()) != 0
-    CCSchemeDataset.get_korea_suspended()
-    assert len(CCSchemeDataset.get_korea_archived()) != 0
+    certified = CCSchemeDataset.get_korea_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    archived = CCSchemeDataset.get_korea_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_singapore():
-    assert len(CCSchemeDataset.get_singapore_certified()) != 0
-    assert len(CCSchemeDataset.get_singapore_archived()) != 0
-    assert len(CCSchemeDataset.get_singapore_in_evaluation()) != 0
+    certified = CCSchemeDataset.get_singapore_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    archived = CCSchemeDataset.get_singapore_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
+    ineval = CCSchemeDataset.get_singapore_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_spain():
-    assert len(CCSchemeDataset.get_spain_certified()) != 0
+    certified = CCSchemeDataset.get_spain_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_sweden():
-    assert len(CCSchemeDataset.get_sweden_certified()) != 0
-    assert len(CCSchemeDataset.get_sweden_in_evaluation()) != 0
-    assert len(CCSchemeDataset.get_sweden_archived()) != 0
+    certified = CCSchemeDataset.get_sweden_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    archived = CCSchemeDataset.get_sweden_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
+    ineval = CCSchemeDataset.get_sweden_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_turkey():
-    assert len(CCSchemeDataset.get_turkey_certified()) != 0
+    certified = CCSchemeDataset.get_turkey_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
 
 
-@pytest.mark.xfail(reason="May fail due to server errors.")
+@pytest.mark.xfail(reason="May fail due to server errors.", raises=RequestException)
 def test_usa():
-    assert len(CCSchemeDataset.get_usa_certified()) != 0
-    assert len(CCSchemeDataset.get_usa_in_evaluation()) != 0
-    assert len(CCSchemeDataset.get_usa_archived()) != 0
+    certified = CCSchemeDataset.get_usa_certified()
+    assert len(certified) != 0
+    assert absolute_urls(certified)
+    archived = CCSchemeDataset.get_usa_archived()
+    assert len(archived) != 0
+    assert absolute_urls(archived)
+    ineval = CCSchemeDataset.get_usa_in_evaluation()
+    assert len(ineval) != 0
+    assert absolute_urls(ineval)