Add extraction of certificate data.

6 files changed, 295 insertions, 47 deletions

diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py
index 398f2435..30a6038a 100644
--- a/src/sec_certs/dataset/cc.py
+++ b/src/sec_certs/dataset/cc.py
@@ -144,6 +144,27 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         return self.targets_dir / "txt"
 
     @property
+    def certificates_dir(self) -> Path:
+        """
+        Returns directory that holds files associated with the certificates
+        """
+        return self.certs_dir / "certificates"
+
+    @property
+    def certificates_pdf_dir(self) -> Path:
+        """
+        Returns directory that holds PDFs associated with certificates
+        """
+        return self.certificates_dir / "pdf"
+
+    @property
+    def certificates_txt_dir(self) -> Path:
+        """
+        Returns directory that holds TXTs associated with certificates
+        """
+        return self.certificates_dir / "txt"
+
+    @property
     def pp_dataset_path(self) -> Path:
         """
         Returns a path to the dataset of Protection Profiles
@@ -242,7 +263,14 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             self.auxiliary_datasets.mu_dset.root_dir = self.mu_dataset_dir
 
         for cert in self:
-            cert.set_local_paths(self.reports_pdf_dir, self.targets_pdf_dir, self.reports_txt_dir, self.targets_txt_dir)
+            cert.set_local_paths(
+                self.reports_pdf_dir,
+                self.targets_pdf_dir,
+                self.certificates_pdf_dir,
+                self.reports_txt_dir,
+                self.targets_txt_dir,
+                self.certificates_txt_dir,
+            )
         # TODO: This forgets to set local paths for other auxiliary datasets
 
     def _merge_certs(self, certs: dict[str, CCCertificate], cert_source: str | None = None) -> None:
@@ -531,6 +559,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     def _download_all_artifacts_body(self, fresh: bool = True) -> None:
         self._download_reports(fresh)
         self._download_targets(fresh)
+        self._download_certs(fresh)
 
     @staged(logger, "Downloading PDFs of CC certification reports.")
     def _download_reports(self, fresh: bool = True) -> None:
@@ -551,7 +580,7 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
     @staged(logger, "Downloading PDFs of CC security targets.")
     def _download_targets(self, fresh: bool = True) -> None:
         self.targets_pdf_dir.mkdir(parents=True, exist_ok=True)
-        certs_to_process = [x for x in self if x.state.report_is_ok_to_download(fresh)]
+        certs_to_process = [x for x in self if x.state.st_is_ok_to_download(fresh)]
 
         if not fresh and certs_to_process:
             logger.info(
@@ -564,6 +593,22 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Downloading PDFs of CC security targets",
         )
 
+    @staged(logger, "Downloading PDFs of CC certificates.")
+    def _download_certs(self, fresh: bool = True) -> None:
+        self.certificates_pdf_dir.mkdir(parents=True, exist_ok=True)
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_download(fresh)]
+
+        if not fresh and certs_to_process:
+            logger.info(
+                f"Downloading {len(certs_to_process)} PDFs of CC certificates for which previous download failed.."
+            )
+
+        cert_processing.process_parallel(
+            CCCertificate.download_pdf_cert,
+            certs_to_process,
+            progress_bar_desc="Downloading PDFs of CC certificates",
+        )
+
     @staged(logger, "Converting PDFs of certification reports to txt.")
     def _convert_reports_to_txt(self, fresh: bool = True) -> None:
         self.reports_txt_dir.mkdir(parents=True, exist_ok=True)
@@ -598,9 +643,28 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
             progress_bar_desc="Converting PDFs of security targets to txt",
         )
 
+    @staged(logger, "Converting PDFs of certificates to txt.")
+    def _convert_certs_to_txt(self, fresh: bool = True) -> None:
+        self.certificates_txt_dir.mkdir(parents=True, exist_ok=True)
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_convert(fresh)]
+
+        if fresh:
+            logger.info("Converting PDFs of certificates to txt.")
+        if not fresh and certs_to_process:
+            logger.info(
+                f"Converting {len(certs_to_process)} PDFs of certificates to txt for which previous conversion failed."
+            )
+
+        cert_processing.process_parallel(
+            CCCertificate.convert_cert_pdf,
+            certs_to_process,
+            progress_bar_desc="Converting PDFs of security targets to txt",
+        )
+
     def _convert_all_pdfs_body(self, fresh: bool = True) -> None:
         self._convert_reports_to_txt(fresh)
         self._convert_targets_to_txt(fresh)
+        self._convert_certs_to_txt(fresh)
 
     @staged(logger, "Extracting report metadata")
     def _extract_report_metadata(self) -> None:
@@ -624,6 +688,17 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
+    @staged(logger, "Extracting cert metadata")
+    def _extract_cert_metadata(self) -> None:
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_analyze()]
+        processed_certs = cert_processing.process_parallel(
+            CCCertificate.extract_cert_pdf_metadata,
+            certs_to_process,
+            use_threading=False,
+            progress_bar_desc="Extracting cert metadata",
+        )
+        self.update_with_certs(processed_certs)
+
     def _extract_pdf_metadata(self) -> None:
         self._extract_report_metadata()
         self._extract_target_metadata()
@@ -639,20 +714,9 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
-    @staged(logger, "Extracting target frontpages")
-    def _extract_target_frontpage(self) -> None:
-        certs_to_process = [x for x in self if x.state.st_is_ok_to_analyze()]
-        processed_certs = cert_processing.process_parallel(
-            CCCertificate.extract_st_pdf_frontpage,
-            certs_to_process,
-            use_threading=False,
-            progress_bar_desc="Extracting target frontpages",
-        )
-        self.update_with_certs(processed_certs)
-
     def _extract_pdf_frontpage(self) -> None:
         self._extract_report_frontpage()
-        self._extract_target_frontpage()
+        # We have no frontpage extraction for targets or certificates themselves, only for the reports.
 
     @staged(logger, "Extracting report keywords")
     def _extract_report_keywords(self) -> None:
@@ -676,9 +740,21 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable
         )
         self.update_with_certs(processed_certs)
 
+    @staged(logger, "Extracting cert keywords")
+    def _extract_cert_keywords(self) -> None:
+        certs_to_process = [x for x in self if x.state.cert_is_ok_to_analyze()]
+        processed_certs = cert_processing.process_parallel(
+            CCCertificate.extract_cert_pdf_keywords,
+            certs_to_process,
+            use_threading=False,
+            progress_bar_desc="Extracting cert keywords",
+        )
+        self.update_with_certs(processed_certs)
+
     def _extract_pdf_keywords(self) -> None:
         self._extract_report_keywords()
         self._extract_target_keywords()
+        self._extract_cert_keywords()
 
     def extract_data(self) -> None:
         logger.info("Extracting various data from certification artifacts")
diff --git a/src/sec_certs/sample/cc.py b/src/sec_certs/sample/cc.py
index 904c794d..a4426d01 100644
--- a/src/sec_certs/sample/cc.py
+++ b/src/sec_certs/sample/cc.py
@@ -104,51 +104,71 @@ class CCCertificate(
 
         st_download_ok: bool  # Whether target download went OK
         report_download_ok: bool  # Whether report download went OK
+        cert_download_ok: bool  # Whether certificate download went OK
         st_convert_garbage: bool  # Whether initial target conversion resulted in garbage
         report_convert_garbage: bool  # Whether initial report conversion resulted in garbage
+        cert_convert_garbage: bool  # Whether initial certificate conversion resulted in garbage
         st_convert_ok: bool  # Whether overall target conversion went OK (either pdftotext or via OCR)
         report_convert_ok: bool  # Whether overall report conversion went OK (either pdftotext or via OCR)
+        cert_convert_ok: bool  # Whether overall certificate conversion went OK (either pdftotext or via OCR)
         st_extract_ok: bool  # Whether target extraction went OK
         report_extract_ok: bool  # Whether report extraction went OK
+        cert_extract_ok: bool  # Whether certificate extraction went OK
 
         st_pdf_hash: str | None
         report_pdf_hash: str | None
+        cert_pdf_hash: str | None
         st_txt_hash: str | None
         report_txt_hash: str | None
+        cert_txt_hash: str | None
 
         _st_pdf_path: Path | None = None
         _report_pdf_path: Path | None = None
+        _cert_pdf_path: Path | None = None
         _st_txt_path: Path | None = None
         _report_txt_path: Path | None = None
+        _cert_txt_path: Path | None = None
 
         def __init__(
             self,
             st_download_ok: bool = False,
             report_download_ok: bool = False,
+            cert_download_ok: bool = False,
             st_convert_garbage: bool = False,
             report_convert_garbage: bool = False,
+            cert_convert_garbage: bool = False,
             st_convert_ok: bool = False,
             report_convert_ok: bool = False,
+            cert_convert_ok: bool = False,
             st_extract_ok: bool = False,
             report_extract_ok: bool = False,
+            cert_extract_ok: bool = False,
             st_pdf_hash: str | None = None,
             report_pdf_hash: str | None = None,
+            cert_pdf_hash: str | None = None,
             st_txt_hash: str | None = None,
             report_txt_hash: str | None = None,
+            cert_txt_hash: str | None = None,
         ):
             super().__init__()
             self.st_download_ok = st_download_ok
             self.report_download_ok = report_download_ok
+            self.cert_download_ok = cert_download_ok
             self.st_convert_garbage = st_convert_garbage
             self.report_convert_garbage = report_convert_garbage
+            self.cert_convert_garbage = cert_convert_garbage
             self.st_convert_ok = st_convert_ok
             self.report_convert_ok = report_convert_ok
+            self.cert_convert_ok = cert_convert_ok
             self.st_extract_ok = st_extract_ok
             self.report_extract_ok = report_extract_ok
+            self.cert_extract_ok = cert_extract_ok
             self.st_pdf_hash = st_pdf_hash
             self.report_pdf_hash = report_pdf_hash
+            self.cert_pdf_hash = cert_pdf_hash
             self.st_txt_hash = st_txt_hash
             self.report_txt_hash = report_txt_hash
+            self.cert_txt_hash = cert_txt_hash
 
         @property
         def st_pdf_path(self) -> Path:
@@ -171,6 +191,16 @@ class CCCertificate(
             self._report_pdf_path = Path(pth) if pth else None
 
         @property
+        def cert_pdf_path(self) -> Path:
+            if not self._cert_pdf_path:
+                raise ValueError(f"cert_pdf_path not set on {type(self)}")
+            return self._cert_pdf_path
+
+        @cert_pdf_path.setter
+        def cert_pdf_path(self, pth: str | Path | None) -> None:
+            self._cert_pdf_path = Path(pth) if pth else None
+
+        @property
         def st_txt_path(self) -> Path:
             if not self._st_txt_path:
                 raise ValueError(f"st_txt_path not set on {type(self)}")
@@ -191,20 +221,36 @@ class CCCertificate(
             self._report_txt_path = Path(pth) if pth else None
 
         @property
+        def cert_txt_path(self) -> Path:
+            if not self._cert_txt_path:
+                raise ValueError(f"cert_txt_path not set on {type(self)}")
+            return self._cert_txt_path
+
+        @cert_txt_path.setter
+        def cert_txt_path(self, pth: str | Path | None) -> None:
+            self._cert_txt_path = Path(pth) if pth else None
+
+        @property
         def serialized_attributes(self) -> list[str]:
             return [
                 "st_download_ok",
                 "report_download_ok",
+                "cert_download_ok",
                 "st_convert_garbage",
                 "report_convert_garbage",
+                "cert_convert_garbage",
                 "st_convert_ok",
                 "report_convert_ok",
+                "cert_convert_ok",
                 "st_extract_ok",
                 "report_extract_ok",
+                "cert_extract_ok",
                 "st_pdf_hash",
                 "report_pdf_hash",
+                "cert_pdf_hash",
                 "st_txt_hash",
                 "report_txt_hash",
+                "cert_txt_hash",
             ]
 
         def report_is_ok_to_download(self, fresh: bool = True) -> bool:
@@ -213,12 +259,18 @@ class CCCertificate(
         def st_is_ok_to_download(self, fresh: bool = True) -> bool:
             return True if fresh else not self.st_download_ok
 
+        def cert_is_ok_to_download(self, fresh: bool = True) -> bool:
+            return True if fresh else not self.st_download_ok
+
         def report_is_ok_to_convert(self, fresh: bool = True) -> bool:
             return self.report_download_ok if fresh else self.report_download_ok and not self.report_convert_ok
 
         def st_is_ok_to_convert(self, fresh: bool = True) -> bool:
             return self.st_download_ok if fresh else self.st_download_ok and not self.st_convert_ok
 
+        def cert_is_ok_to_convert(self, fresh: bool = True) -> bool:
+            return self.cert_download_ok if fresh else self.cert_download_ok and not self.cert_convert_ok
+
         def report_is_ok_to_analyze(self, fresh: bool = True) -> bool:
             if fresh is True:
                 return self.report_download_ok and self.report_convert_ok
@@ -231,6 +283,12 @@ class CCCertificate(
             else:
                 return self.st_download_ok and self.st_convert_ok and not self.st_extract_ok
 
+        def cert_is_ok_to_analyze(self, fresh: bool = True) -> bool:
+            if fresh is True:
+                return self.cert_download_ok and self.cert_convert_ok
+            else:
+                return self.cert_download_ok and self.cert_convert_ok and not self.cert_extract_ok
+
     @dataclass
     class PdfData(BasePdfData, ComplexSerializableType):
         """
@@ -239,12 +297,16 @@ class CCCertificate(
 
         report_metadata: dict[str, Any] | None = field(default=None)
         st_metadata: dict[str, Any] | None = field(default=None)
+        cert_metadata: dict[str, Any] | None = field(default=None)
         report_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
         st_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
+        cert_frontpage: dict[str, dict[str, Any]] | None = field(default=None)
         report_keywords: dict[str, Any] | None = field(default=None)
         st_keywords: dict[str, Any] | None = field(default=None)
+        cert_keywords: dict[str, Any] | None = field(default=None)
         report_filename: str | None = field(default=None)
         st_filename: str | None = field(default=None)
+        cert_filename: str | None = field(default=None)
 
         def __bool__(self) -> bool:
             return any(x is not None for x in vars(self))
@@ -814,25 +876,33 @@ class CCCertificate(
         self,
         report_pdf_dir: str | Path | None,
         st_pdf_dir: str | Path | None,
+        cert_pdf_dir: str | Path | None,
         report_txt_dir: str | Path | None,
         st_txt_dir: str | Path | None,
+        cert_txt_dir: str | Path | None,
     ) -> None:
         """
         Sets paths to files given the requested directories
 
         :param Optional[Union[str, Path]] report_pdf_dir: Directory where pdf reports shall be stored
         :param Optional[Union[str, Path]] st_pdf_dir: Directory where pdf security targets shall be stored
+        :param Optional[Union[str, Path]] cert_pdf_dir: Directory where pdf certificates shall be stored
         :param Optional[Union[str, Path]] report_txt_dir: Directory where txt reports shall be stored
         :param Optional[Union[str, Path]] st_txt_dir: Directory where txt security targets shall be stored
+        :param Optional[Union[str, Path]] cert_txt_dir: Directory where txtcertificates shall be stored
         """
         if report_pdf_dir:
             self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + ".pdf")
         if st_pdf_dir:
             self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + ".pdf")
+        if cert_pdf_dir:
+            self.state.cert_pdf_path = Path(cert_pdf_dir) / (self.dgst + ".pdf")
         if report_txt_dir:
             self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + ".txt")
         if st_txt_dir:
             self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + ".txt")
+        if cert_txt_dir:
+            self.state.cert_txt_path = Path(cert_txt_dir) / (self.dgst + ".txt")
 
     @staticmethod
     def download_pdf_report(cert: CCCertificate) -> CCCertificate:
@@ -880,11 +950,33 @@ class CCCertificate(
         return cert
 
     @staticmethod
+    def download_pdf_cert(cert: CCCertificate) -> CCCertificate:
+        """
+        Downloads pdf of the certificate. Staticmethod to allow for parallelization.
+
+        :param CCCertificate cert: cert to download the pdf of
+        :return CCCertificate: returns the modified certificate with updated state
+        """
+        exit_code: str | int = (
+            helpers.download_file(cert.cert_link, cert.state.cert_pdf_path) if cert.cert_link else "No link"
+        )
+
+        if exit_code != requests.codes.ok:
+            error_msg = f"failed to download certificate from {cert.cert_link}, code: {exit_code}"
+            logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
+            cert.state.cert_download_ok = False
+        else:
+            cert.state.cert_download_ok = True
+            cert.state.cert_pdf_hash = helpers.get_sha256_filepath(cert.state.cert_pdf_path)
+            cert.pdf_data.cert_filename = unquote_plus(str(urlparse(cert.cert_link).path).split("/")[-1])
+        return cert
+
+    @staticmethod
     def convert_report_pdf(cert: CCCertificate) -> CCCertificate:
         """
         Converts the pdf certification report to txt, given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to download the pdf report for
+        :param CCCertificate cert: cert to convert the pdf report for
         :return CCCertificate: the modified certificate with updated state
         """
         ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(
@@ -906,7 +998,7 @@ class CCCertificate(
         """
         Converts the pdf security target to txt, given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to download the pdf security target for
+        :param CCCertificate cert: cert to convert the pdf security target for
         :return CCCertificate: the modified certificate with updated state
         """
         ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path)
@@ -922,18 +1014,23 @@ class CCCertificate(
         return cert
 
     @staticmethod
-    def extract_st_pdf_metadata(cert: CCCertificate) -> CCCertificate:
+    def convert_cert_pdf(cert: CCCertificate) -> CCCertificate:
         """
-        Extracts metadata from security target pdf given the certificate. Staticmethod to allow for parallelization.
+        Converts the pdf certificate to txt, given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to extract the metadata for.
+        :param CCCertificate cert: cert to convert the certificate for
         :return CCCertificate: the modified certificate with updated state
         """
-        response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path)
-        if response != constants.RETURNCODE_OK:
-            cert.state.st_extract_ok = False
+        ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.cert_pdf_path, cert.state.cert_txt_path)
+        # If OCR was done the result was garbage
+        cert.state.cert_convert_garbage = ocr_done
+        # And put the whole result into convert_ok
+        cert.state.cert_convert_ok = ok_result
+        if not ok_result:
+            error_msg = "failed to convert security target pdf->txt"
+            logger.error(f"Cert dgst: {cert.dgst} " + error_msg)
         else:
-            cert.state.st_extract_ok = True
+            cert.state.cert_txt_hash = helpers.get_sha256_filepath(cert.state.cert_txt_path)
         return cert
 
     @staticmethod
@@ -952,20 +1049,33 @@ class CCCertificate(
         return cert
 
     @staticmethod
-    def extract_st_pdf_frontpage(cert: CCCertificate) -> CCCertificate:
+    def extract_st_pdf_metadata(cert: CCCertificate) -> CCCertificate:
         """
-        Extracts data from security target pdf frontpage given the certificate. Staticmethod to allow for parallelization.
+        Extracts metadata from security target pdf given the certificate. Staticmethod to allow for parallelization.
 
-        :param CCCertificate cert: cert to extract the frontpage data for.
+        :param CCCertificate cert: cert to extract the metadata for.
         :return CCCertificate: the modified certificate with updated state
         """
-        cert.pdf_data.st_frontpage = {}
+        response, cert.pdf_data.st_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.st_pdf_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.st_extract_ok = False
+        else:
+            cert.state.st_extract_ok = True
+        return cert
 
-        for header_type, associated_header_func in HEADERS.items():
-            response, cert.pdf_data.st_frontpage[header_type] = associated_header_func(cert.state.st_txt_path)
+    @staticmethod
+    def extract_cert_pdf_metadata(cert: CCCertificate) -> CCCertificate:
+        """
+        Extracts metadata from certificate pdf given the certificate. Staticmethod to allow for parallelization.
 
-            if response != constants.RETURNCODE_OK:
-                cert.state.st_extract_ok = False
+        :param CCCertificate cert: cert to extract the metadata for.
+        :return CCCertificate: the modified certificate with updated state
+        """
+        response, cert.pdf_data.cert_metadata = sec_certs.utils.pdf.extract_pdf_metadata(cert.state.cert_pdf_path)
+        if response != constants.RETURNCODE_OK:
+            cert.state.cert_extract_ok = False
+        else:
+            cert.state.cert_extract_ok = True
         return cert
 
     @staticmethod
@@ -988,7 +1098,7 @@ class CCCertificate(
     @staticmethod
     def extract_report_pdf_keywords(cert: CCCertificate) -> CCCertificate:
         """
-        Matches regular expresions in txt obtained from certification report and extracts the matches into attribute.
+        Matches regular expressions in txt obtained from certification report and extracts the matches into attribute.
         Static method to allow for parallelization
 
         :param CCCertificate cert: certificate to extract the keywords for.
@@ -1004,7 +1114,7 @@ class CCCertificate(
     @staticmethod
     def extract_st_pdf_keywords(cert: CCCertificate) -> CCCertificate:
         """
-        Matches regular expresions in txt obtained from security target and extracts the matches into attribute.
+        Matches regular expressions in txt obtained from security target and extracts the matches into attribute.
         Static method to allow for parallelization
 
         :param CCCertificate cert: certificate to extract the keywords for.
@@ -1017,6 +1127,22 @@ class CCCertificate(
             cert.pdf_data.st_keywords = st_keywords
         return cert
 
+    @staticmethod
+    def extract_cert_pdf_keywords(cert: CCCertificate) -> CCCertificate:
+        """
+        Matches regular expressions in txt obtained from the certificate and extracts the matches into attribute.
+        Static method to allow for parallelization
+
+        :param CCCertificate cert: certificate to extract the keywords for.
+        :return CCCertificate: the modified certificate with extracted keywords.
+        """
+        cert_keywords = sec_certs.utils.extract.extract_keywords(cert.state.cert_txt_path, cc_rules)
+        if cert_keywords is None:
+            cert.state.cert_extract_ok = False
+        else:
+            cert.pdf_data.cert_keywords = cert_keywords
+        return cert
+
     def compute_heuristics_version(self) -> None:
         """
         Fills in the heuristically obtained version of certified product into attribute in heuristics class.
diff --git a/tests/cc/test_cc_certificate.py b/tests/cc/test_cc_certificate.py
index bdc10f32..32bd3a59 100644
--- a/tests/cc/test_cc_certificate.py
+++ b/tests/cc/test_cc_certificate.py
@@ -42,10 +42,6 @@ def test_extract_metadata(vulnerable_certificate: CCCertificate):
 
 
 def test_extract_frontpage(vulnerable_certificate: CCCertificate):
-    vulnerable_certificate.state.st_extract_ok = True
-    CCCertificate.extract_st_pdf_frontpage(vulnerable_certificate)
-    assert vulnerable_certificate.state.st_extract_ok
-
     vulnerable_certificate.state.report_extract_ok = True
     CCCertificate.extract_report_pdf_frontpage(vulnerable_certificate)
     assert vulnerable_certificate.state.report_extract_ok
diff --git a/tests/data/cc/certificate/fictional_cert.json b/tests/data/cc/certificate/fictional_cert.json
index ff95795c..96f42c07 100644
--- a/tests/data/cc/certificate/fictional_cert.json
+++ b/tests/data/cc/certificate/fictional_cert.json
@@ -43,27 +43,37 @@
         "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
         "st_download_ok": false,
         "report_download_ok": false,
+        "cert_download_ok": false,
         "st_convert_garbage": false,
         "report_convert_garbage": false,
+        "cert_convert_garbage": false,
         "st_convert_ok": false,
         "report_convert_ok": false,
+        "cert_convert_ok": false,
         "st_extract_ok": false,
         "report_extract_ok": false,
+        "cert_extract_ok": false,
         "st_pdf_hash": null,
         "report_pdf_hash": null,
+        "cert_pdf_hash": null,
         "st_txt_hash": null,
-        "report_txt_hash": null
+        "report_txt_hash": null,
+        "cert_txt_hash": null
     },
     "pdf_data": {
         "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
         "report_metadata": null,
         "st_metadata": null,
+        "cert_metadata": null,
         "report_frontpage": null,
         "st_frontpage": null,
+        "cert_frontpage": null,
         "report_keywords": null,
         "st_keywords": null,
+        "cert_keywords": null,
         "report_filename": null,
-        "st_filename": null
+        "st_filename": null,
+        "cert_filename": null
     },
     "heuristics": {
         "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
diff --git a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
index 4c1ff4c7..bc720a76 100644
--- a/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
+++ b/tests/data/cc/dataset/auxiliary_datasets/maintenances/maintenance_updates.json
@@ -24,27 +24,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": true,
                 "report_download_ok": true,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": "d42e4364d037ba742fcd4050a9a84d0e6300f93eb68bcfe8c61f72c429c9ceca",
                 "report_pdf_hash": "80bada65614c1b037c13efa78996a8910700d0e05a3ca217286f76d7dacefe62",
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": "383-7-159 MR v1.0e.pdf",
-                "st_filename": "383-7-159 ST v1.4 CCRA.pdf"
+                "st_filename": "383-7-159 ST v1.4 CCRA.pdf",
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
diff --git a/tests/data/cc/dataset/toy_dataset.json b/tests/data/cc/dataset/toy_dataset.json
index e32cffa4..e60ecf2c 100644
--- a/tests/data/cc/dataset/toy_dataset.json
+++ b/tests/data/cc/dataset/toy_dataset.json
@@ -47,27 +47,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": false,
                 "report_download_ok": false,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": null,
                 "report_pdf_hash": null,
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
@@ -136,27 +146,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": false,
                 "report_download_ok": false,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": null,
                 "report_pdf_hash": null,
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",
@@ -233,27 +253,37 @@
                 "_type": "sec_certs.sample.cc.CCCertificate.InternalState",
                 "st_download_ok": false,
                 "report_download_ok": false,
+                "cert_download_ok": false,
                 "st_convert_garbage": false,
                 "report_convert_garbage": false,
+                "cert_convert_garbage": false,
                 "st_convert_ok": false,
                 "report_convert_ok": false,
+                "cert_convert_ok": false,
                 "st_extract_ok": false,
                 "report_extract_ok": false,
+                "cert_extract_ok": false,
                 "st_pdf_hash": null,
                 "report_pdf_hash": null,
+                "cert_pdf_hash": null,
                 "st_txt_hash": null,
-                "report_txt_hash": null
+                "report_txt_hash": null,
+                "cert_txt_hash": null
             },
             "pdf_data": {
                 "_type": "sec_certs.sample.cc.CCCertificate.PdfData",
                 "report_metadata": null,
                 "st_metadata": null,
+                "cert_metadata": null,
                 "report_frontpage": null,
                 "st_frontpage": null,
+                "cert_frontpage": null,
                 "report_keywords": null,
                 "st_keywords": null,
+                "cert_keywords": null,
                 "report_filename": null,
-                "st_filename": null
+                "st_filename": null,
+                "cert_filename": null
             },
             "heuristics": {
                 "_type": "sec_certs.sample.cc.CCCertificate.Heuristics",