Fix Malaysia scheme download.

3 files changed, 42 insertions, 51 deletions

diff --git a/sec_certs/constants.py b/sec_certs/constants.py
index f6da500c..d791d3fb 100644
--- a/sec_certs/constants.py
+++ b/sec_certs/constants.py
@@ -76,9 +76,14 @@ CC_ITALY_INEVAL_URL = CC_ITALY_BASE_URL + "/index.php/elenchi-certificazioni/in-
 CC_JAPAN_CERTIFIED_URL = "https://www.ipa.go.jp/security/jisec/jisec_e/certified_products/certfy_list_e31.html"
 CC_JAPAN_ARCHIVED_URL = "https://www.ipa.go.jp/security/jisec/jisec_e/certified_products/certfy_list_e_archive.html"
 CC_JAPAN_INEVAL_URL = "https://www.ipa.go.jp/security/jisec/jisec_e/prdct_in_eval.html"
-CC_MALAYSIA_BASE_URL = "https://www.cybersecurity.my/mycc"
-CC_MALAYSIA_CERTIFIED_URL = CC_MALAYSIA_BASE_URL + "/mycprA.html"
-CC_MALAYSIA_INEVAL_URL = CC_MALAYSIA_BASE_URL + "/mycprC.html"
+CC_MALAYSIA_BASE_URL = "https://iscb.cybersecurity.my"
+CC_MALAYSIA_CERTIFIED_URL = (
+    CC_MALAYSIA_BASE_URL + "/en/index.php/certification/product-certification/mycc/certified-products-and-systems"
+)
+CC_MALAYSIA_INEVAL_URL = (
+    CC_MALAYSIA_BASE_URL
+    + "/en/index.php/certification/product-certification/mycc/list-of-products-and-systems-under-evaluation-or-maintenance"
+)
 CC_NETHERLANDS_BASE_URL = "https://www.tuv-nederland.nl/common-criteria"
 CC_NETHERLANDS_CERTIFIED_URL = CC_NETHERLANDS_BASE_URL + "/certificates.html"
 CC_NETHERLANDS_INEVAL_URL = CC_NETHERLANDS_BASE_URL + "/ongoing-certifications.html"
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index 24d8bc41..378bca61 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -1366,61 +1366,47 @@ class CCSchemeDataset:
     @staticmethod
     def get_malaysia_certified():
         soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_CERTIFIED_URL)
-        main_table = soup.find("table").find("table")
-        cert_table = main_table.find_all("table")[1].find("table")
-        tables = cert_table.find_all(lambda x: x.name == "table" and x.find_parent("table") == cert_table)
-        category_name = None
-        archive = False
+        main_div = soup.find("div", attrs={"itemprop": "articleBody"})
+        tables = main_div.find_all("table", recursive=False)
         results = []
         for table in tables:
-            if table.find("table") is None:
-                name_td = table.find_all("td")[1]
-                if "Archive" in name_td.text:
-                    category_name = str(name_td.text).split("Archive")[1]
-                    archive = True
-                else:
-                    category_name = str(name_td.text)
-                    archive = False
-                continue
-            data_table = table.find("table")
-            tr = data_table.find_all("tr")[1]
-            tds = tr.find_all("td")
-            if len(tds) != 6:
-                continue
-            cert = {
-                "category": category_name,
-                "archived": archive,
-                "level": str(tds[0].text),
-                "cert_id": str(tds[1].text),
-                "certification_date": str(tds[2].text),
-                "product": str(tds[3].text),
-                "developer": str(tds[4].text),
-            }
-            results.append(cert)
+            category_name = str(table.find_previous_sibling("h3").text)
+            for tr in table.find_all("tr")[1:]:
+                tds = tr.find_all("td")
+                if len(tds) != 6:
+                    continue
+                cert = {
+                    "category": category_name,
+                    "level": str(tds[0].text),
+                    "cert_id": str(tds[1].text),
+                    "certification_date": str(tds[2].text),
+                    "product": str(tds[3].text),
+                    "developer": str(tds[4].text),
+                }
+                results.append(cert)
         return results
 
     @staticmethod
-    def get_malaysia_in_evalution():
+    def get_malaysia_in_evaluation():
         soup = CCSchemeDataset._download_page(constants.CC_MALAYSIA_INEVAL_URL)
-        cert_table = soup.find("table").find("table").find_all("table")[1]
-        tables = cert_table.find_all(lambda x: x.name == "table" and x.find_parent("table") == cert_table)
+        main_div = soup.find("div", attrs={"itemprop": "articleBody"})
+        tables = main_div.find_all("table", recursive=False)
         results = []
         for table in tables:
-            cat_p = table.find_previous_sibling("p")
-            data_table = table.find("table")
-            tr = data_table.find_all("tr")[1]
-            tds = tr.find_all("td")
-            if len(tds) != 5:
-                continue
-            cert = {
-                "category": str(cat_p.text),
-                "level": str(tds[0].text),
-                "project_id": str(tds[1].text),
-                "toe_name": str(tds[2].text),
-                "developer": str(tds[3].text),
-                "expected_completion_date": str(tds[4].text),
-            }
-            results.append(cert)
+            category_name = str(table.find_previous_sibling("h3").text)
+            for tr in table.find_all("tr")[1:]:
+                tds = tr.find_all("td")
+                if len(tds) != 5:
+                    continue
+                cert = {
+                    "category": category_name,
+                    "level": str(tds[0].text),
+                    "project_id": str(tds[1].text),
+                    "toe_name": str(tds[2].text),
+                    "developer": str(tds[3].text),
+                    "expected_completion": str(tds[4].text),
+                }
+                results.append(cert)
         return results
 
     @staticmethod
diff --git a/tests/test_cc_schemes.py b/tests/test_cc_schemes.py
index be840edb..b3fa0a9f 100644
--- a/tests/test_cc_schemes.py
+++ b/tests/test_cc_schemes.py
@@ -35,7 +35,7 @@ class TestCCSchemes(TestCase):
 
     def test_malaysia(self):
         assert len(CCSchemeDataset.get_malaysia_certified()) != 0
-        assert len(CCSchemeDataset.get_malaysia_in_evalution()) != 0
+        assert len(CCSchemeDataset.get_malaysia_in_evaluation()) != 0
 
     def test_netherlands(self):
         assert len(CCSchemeDataset.get_netherlands_certified()) != 0