Plug-in old extractions methods to new OOP frame

- Tests for txt file processing still missing
- The internals of the txt processing methods are still authored by petrs
- They are however fully parallel and called in an OOP fashion
- Overall, all results are gathered in a single json file
- All results are implicitly paired

11 files changed, 470 insertions, 87 deletions

diff --git a/cc_oop_demo.py b/cc_oop_demo.py
index a49a2e7d..49729400 100644
--- a/cc_oop_demo.py
+++ b/cc_oop_demo.py
@@ -22,7 +22,7 @@ def main():
     dset = CCDataset({}, Path('./debug_dataset'), 'sample_dataset', 'sample dataset description')
 
     # Load metadata for certificates from CSV and HTML sources
-    dset.get_certs_from_web()
+    dset.get_certs_from_web(to_download=False)
     logger.info(f'Finished parsing. Have dataset with {len(dset)} certificates.')
 
     # # Dump dataset into JSON
@@ -39,6 +39,11 @@ def main():
     # Convert pdfs to text
     dset.convert_all_pdfs()
 
+    # Extract data from txt files
+    dset.extract_data()
+
+    dset.to_json('./debug_dataset/cc_full_dataset.json')
+
     end = datetime.now()
     logger.info(f'The computation took {(end-start)} seconds.')
 
diff --git a/process_certificates.py b/process_certificates.py
index badd84e8..f5bbcf39 100755
--- a/process_certificates.py
+++ b/process_certificates.py
@@ -5,6 +5,7 @@ from sec_certs.files import load_json_files
 from sec_certs.extract_certificates import *
 from sec_certs.analyze_certificates import *
 from sec_certs.download import download_cc_web, download_cc
+from sec_certs.cert_rules import rules as cc_search_rules
 
 
 @click.command()
@@ -118,7 +119,7 @@ def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_ext
         convert_pdf_files(walk_dir, threads, ["-raw"])
 
     if do_extraction_certs:
-        all_keywords = extract_certificates_keywords_parallel(walk_dir, fragments_dir, 'certificate', threads)
+        all_keywords = extract_certificates_keywords_parallel(walk_dir, fragments_dir, 'certificate', cc_search_rules, threads)
         with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file:
             json.dump(all_keywords,  write_file, indent=4, sort_keys=True)
 
diff --git a/sec_certs/certificate.py b/sec_certs/certificate.py
index 04dd41d1..775e3d07 100644
--- a/sec_certs/certificate.py
+++ b/sec_certs/certificate.py
@@ -833,13 +833,24 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
     class PdfData(ComplexSerializableType):
         report_metadata: Dict[str, str]
         st_metadata: Dict[str, str]
+        report_frontpage: Dict[str, str]
+        st_frontpage: Dict[str, str]
+        report_keywords: Dict[str, str]
+        st_keywords: Dict[str, str]
 
-        def __init__(self, report_metadata: Optional[Dict[str, str]] = None, st_metadata: Optional[Dict[str, str]] = None):
+        def __init__(self, report_metadata: Optional[Dict[str, str]] = None, st_metadata: Optional[Dict[str, str]] = None,
+                     report_frontpage: Optional[Dict[str, str]] = None, st_frontpage: Optional[Dict[str, str]] = None,
+                     report_keywords: Optional[Dict[str,str]] = None, st_keywords: Optional[Dict[str, str]] = None):
             self.report_metadata = report_metadata
             self.st_metadata = st_metadata
+            self.report_frontpage = report_frontpage
+            self.st_frontpage = st_frontpage
+            self.report_keywords = report_keywords
+            self.st_keywords = st_keywords
 
         def to_dict(self):
-            return {'report_metadata': self.report_metadata, 'st_metadata': self.st_metadata}
+            return {'report_metadata': self.report_metadata, 'st_metadata': self.st_metadata, 'report_frontpage': self.report_frontpage,
+                    'st_frontpage': self.st_frontpage, 'report_keywords': self.report_keywords, 'st_keywords': self.st_keywords}
 
         @classmethod
         def from_dict(cls, dct: Dict[str, bool]):
@@ -1092,4 +1103,29 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
     @staticmethod
     def extract_report_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
         cert.pdf_data.report_metadata = helpers.extract_pdf_metadata(cert.state.report_pdf_path)[1]
-        return cert
\ No newline at end of file
+        return cert
+
+    @staticmethod
+    def extract_st_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        cert.pdf_data.st_frontpage = dict()
+        cert.pdf_data.st_frontpage['bsi'] = helpers.search_only_headers_bsi(cert.state.st_txt_path)
+        cert.pdf_data.st_frontpage['anssi'] = helpers.search_only_headers_anssi(cert.state.st_txt_path)
+        return cert
+
+    @staticmethod
+    def extract_report_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        cert.pdf_data.report_frontpage = dict()
+        cert.pdf_data.report_frontpage['bsi'] = helpers.search_only_headers_bsi(cert.state.report_txt_path)
+        cert.pdf_data.report_frontpage['anssi'] = helpers.search_only_headers_anssi(cert.state.report_txt_path)
+        return cert
+
+    @staticmethod
+    def extract_report_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        cert.pdf_data.report_keywords = helpers.extract_keywords(cert.state.report_txt_path)
+        return cert
+
+    @staticmethod
+    def extract_st_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert':
+        cert.pdf_data.st_keywords = helpers.extract_keywords(cert.state.st_txt_path)
+        return cert
+
diff --git a/sec_certs/dataset.py b/sec_certs/dataset.py
index 0b0633e9..80eb49bd 100644
--- a/sec_certs/dataset.py
+++ b/sec_certs/dataset.py
@@ -480,7 +480,7 @@ class CCDataset(Dataset, ComplexSerializableType):
         if fresh is True:
             certs_to_process = self.certs.values()
         else:
-            certs_to_process = [x for x in self.certs.values() if not x.state.st_extract_ok]
+            certs_to_process = [x for x in self.certs.values() if not x.state.report_extract_ok]
         cert_processing.process_parallel(CommonCriteriaCert.extract_report_pdf_metadata, certs_to_process, constants.N_THREADS)
 
     def _extract_targets_metadata(self, fresh: bool = True):
@@ -495,15 +495,61 @@ class CCDataset(Dataset, ComplexSerializableType):
         self._extract_report_metadata(fresh)
         self._extract_targets_metadata(fresh)
 
-    def extract_all_keywords(self, fresh: bool = True):
-        pass
+    def _extract_targets_frontpage(self, fresh: bool = True):
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.st_extract_ok]
+        cert_processing.process_parallel(CommonCriteriaCert.extract_st_pdf_frontpage, certs_to_process, constants.N_THREADS)
+
+    def _extract_report_frontpage(self, fresh: bool = True):
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.report_extract_ok]
+        cert_processing.process_parallel(CommonCriteriaCert.extract_report_pdf_frontpage, certs_to_process, constants.N_THREADS)
 
     def extract_pdf_frontpage(self, fresh: bool = True):
-        pass
+        logger.info('Extracting pdf frontpages from CC dataset.')
+        self._extract_report_frontpage(fresh)
+        self._extract_targets_frontpage(fresh)
 
-    def extract_data(self, fresh:bool = True):
-        # TODO: Extract metadata, keywords, frontpage, attempt to re-do once if fails
-        pass
+    def _extract_report_keywords(self, fresh: bool = True):
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.report_extract_ok]
+        cert_processing.process_parallel(CommonCriteriaCert.extract_report_pdf_keywords, certs_to_process, constants.N_THREADS)
+
+    def _extract_targets_keywords(self, fresh: bool = True):
+        if fresh is True:
+            certs_to_process = self.certs.values()
+        else:
+            certs_to_process = [x for x in self.certs.values() if not x.state.st_extract_ok]
+        cert_processing.process_parallel(CommonCriteriaCert.extract_st_pdf_keywords, certs_to_process, constants.N_THREADS)
+
+    def extract_pdf_keywords(self, fresh: bool = True):
+        logger.info('Extracting pdf keywords from CC dataset.')
+        self._extract_report_keywords(fresh)
+        self._extract_targets_keywords(fresh)
+
+    def extract_data(self, fresh: bool = True):
+        logger.info('Extracting various stuff from converted txt filed from CC dataset.')
+        self.extract_pdf_metadata(fresh)
+        self.extract_pdf_frontpage(fresh)
+        self.extract_pdf_keywords(fresh)
+
+        if fresh is True:
+            if any(filter(lambda x: not x.state.report_extract_ok, self.certs.values())):
+                logger.info('Attempting to re-extract failed data from report txts')
+                self._extract_report_metadata(False)
+                self._extract_report_frontpage(False)
+                self._extract_report_keywords(False)
+            if any(filter(lambda x: not x.state.st_extract_ok, self.certs.values())):
+                logger.info('Attempting to re-extract failed data from ST txts')
+                self._extract_targets_metadata(False)
+                self._extract_targets_frontpage(False)
+                self._extract_targets_keywords(False)
 
 
 class FIPSDataset(Dataset, ComplexSerializableType):
diff --git a/sec_certs/extract_certificates.py b/sec_certs/extract_certificates.py
index aada02b7..7de9a053 100644
--- a/sec_certs/extract_certificates.py
+++ b/sec_certs/extract_certificates.py
@@ -24,6 +24,10 @@ from .cert_rules import rules, fips_rules, REGEXEC_SEP
 from .files import search_files, load_cert_html_file, FILE_ERRORS_STRATEGY
 from .constants import *
 
+
+import sec_certs.constants as constants
+import sec_certs.helpers as helpers
+
 plt.rcdefaults()
 
 # if True, then exception is raised when unexpect intermediate number is obtained
@@ -416,11 +420,11 @@ def search_only_headers_bsi(walk_dir: Path):
                     if end_pos != -1:
                         developer = developer[:end_pos]
 
-                    items_found[const.TAG_CERT_ID] = normalize_match_string(cert_id)
-                    items_found[const.TAG_CERT_ITEM] = normalize_match_string(
+                    items_found[constants.TAG_CERT_ID] = normalize_match_string(cert_id)
+                    items_found[constants.TAG_CERT_ITEM] = normalize_match_string(
                         certified_item)
-                    items_found[const.TAG_DEVELOPER] = normalize_match_string(developer)
-                    items_found[const.TAG_CERT_LAB] = 'BSI'
+                    items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer)
+                    items_found[constants.TAG_CERT_LAB] = 'BSI'
 
             #
             # Process page with more detailed certificate info
diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py
index f843a121..bbb1be42 100644
--- a/sec_certs/helpers.py
+++ b/sec_certs/helpers.py
@@ -14,9 +14,13 @@ import numpy as np
 import pandas as pd
 import subprocess
 import sec_certs.constants as constants
+from enum import Enum
 
 from PyPDF2 import PdfFileReader
 
+import sec_certs.extract_certificates as extract_certificates
+from sec_certs.cert_rules import REGEXEC_SEP
+from sec_certs.cert_rules import rules as cc_search_rules
 
 logger = logging.getLogger(__name__)
 
@@ -166,7 +170,8 @@ def repair_pdf(file: Path):
 
 
 def convert_pdf_file(pdf_path: Path, txt_path: Path, options):
-    return subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60).returncode
+    return subprocess.run(['pdftotext', *options, pdf_path, txt_path], stdout=subprocess.DEVNULL,
+                          stderr=subprocess.DEVNULL, timeout=60).returncode
 
 
 def extract_pdf_metadata(filepath: Path):
@@ -187,3 +192,273 @@ def extract_pdf_metadata(filepath: Path):
         logger.error(f'Failed to read metadata of {filepath}, error: {e}')
 
     return filepath, metadata
+
+
+# TODO: Please, refactor me. I reallyyyyyyyyyyyyy need it!!!!!!
+def search_only_headers_anssi(filepath: Path):
+    class HEADER_TYPE(Enum):
+        HEADER_FULL = 1
+        HEADER_MISSING_CERT_ITEM_VERSION = 2
+        HEADER_MISSING_PROTECTION_PROFILES = 3
+        HEADER_DUPLICITIES = 4
+
+    rules_certificate_preface = [
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur\(s\)(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d\’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit \(référence/version\)(.+)Nom de la TOE \(référence/version\)(.+)Conformité à un profil de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur\(s\)(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit \(référence/version\)(.+)Nom de la TOE \(référence/version\)(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit \(référence/version\)(.+)Nom de la TOE \(référence/version\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\(s\)(.+)Evaluation facility(.+)Recognition arrangements'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification report reference(.+)Product name \(reference / version\)(.+)TOE name \(reference / version\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'),
+        (HEADER_TYPE.HEADER_FULL,
+         'Certification report reference(.+)TOE name(.+)Product\'s reference/ version(.+)TOE\'s reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements'),
+
+        # corrupted text (duplicities)
+        (HEADER_TYPE.HEADER_DUPLICITIES,
+         'Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles'),
+
+        # rules without product version
+        (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
+         'Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
+         'Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+        (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+
+        # rules without protection profile
+        (HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES,
+         'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'),
+    ]
+
+    # statistics about rules success rate
+    num_rules_hits = {}
+    for rule in rules_certificate_preface:
+        num_rules_hits[rule[1]] = 0
+
+    items_found = {}
+    whole_text, whole_text_with_newlines, was_unicode_decode_error = extract_certificates.load_cert_file(filepath)
+
+    # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs
+    pos = whole_text.find('')
+    if pos != -1:
+        pos = whole_text.find('', pos)
+        if pos != -1:
+            whole_text = whole_text[pos:]
+
+    no_match_yet = True
+    other_rule_already_match = False
+    rule_index = -1
+    for rule in rules_certificate_preface:
+        rule_index += 1
+        rule_and_sep = rule[1] + REGEXEC_SEP
+
+        for m in re.finditer(rule_and_sep, whole_text):
+            if no_match_yet:
+                items_found[constants.TAG_HEADER_MATCH_RULES] = []
+                no_match_yet = False
+
+            # insert rule if at least one match for it was found
+            if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]:
+                items_found[constants.TAG_HEADER_MATCH_RULES].append(rule[1])
+
+            if not other_rule_already_match:
+                other_rule_already_match = True
+            else:
+                logger.warning(f'WARNING: multiple rules are matching same certification document: {filepath}')
+
+            num_rules_hits[rule[1]] += 1  # add hit to this rule
+            match_groups = m.groups()
+            index_next_item = 0
+            items_found[constants.TAG_CERT_ID] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+            index_next_item += 1
+
+            items_found[constants.TAG_CERT_ITEM] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+            index_next_item += 1
+
+            if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION:
+                items_found[constants.TAG_CERT_ITEM_VERSION] = ''
+            else:
+                items_found[constants.TAG_CERT_ITEM_VERSION] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+
+            if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES:
+                items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = ''
+            else:
+                items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+                index_next_item += 1
+
+            items_found[constants.TAG_CC_VERSION] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+            index_next_item += 1
+
+            items_found[constants.TAG_CC_SECURITY_LEVEL] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+            index_next_item += 1
+
+            items_found[constants.TAG_DEVELOPER] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+            index_next_item += 1
+
+            items_found[constants.TAG_CERT_LAB] = extract_certificates.normalize_match_string(match_groups[index_next_item])
+            index_next_item += 1
+
+    # if True:
+    #     print('# hits for rule')
+    #     sorted_rules = sorted(num_rules_hits.items(),
+    #                           key=operator.itemgetter(1), reverse=True)
+    #     used_rules = []
+    #     for rule in sorted_rules:
+    #         print('{:4d} : {}'.format(rule[1], rule[0]))
+    #         if rule[1] > 0:
+    #             used_rules.append(rule[0])
+
+    return items_found
+
+# TODO: Please refactor me. I need it so badlyyyyyy!!!
+def search_only_headers_bsi(filepath: Path):
+    LINE_SEPARATOR_STRICT = ' '
+    NUM_LINES_TO_INVESTIGATE = 15
+    rules_certificate_preface = [
+        '(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)',
+        '(BSI-DSZ-CC-.+?) zu (.+?) der (.*)',
+    ]
+
+    items_found = {}
+    no_match_yet = True
+    #
+    # Process front page with info: cert_id, certified_item and developer
+    #
+    whole_text, whole_text_with_newlines, was_unicode_decode_error = extract_certificates.load_cert_file(filepath, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT)
+
+    for rule in rules_certificate_preface:
+        rule_and_sep = rule + REGEXEC_SEP
+
+        for m in re.finditer(rule_and_sep, whole_text):
+            if no_match_yet:
+                items_found[constants.TAG_HEADER_MATCH_RULES] = []
+                no_match_yet = False
+
+            # insert rule if at least one match for it was found
+            if rule not in items_found[constants.TAG_HEADER_MATCH_RULES]:
+                items_found[constants.TAG_HEADER_MATCH_RULES].append(rule)
+
+            match_groups = m.groups()
+            cert_id = match_groups[0]
+            certified_item = match_groups[1]
+            developer = match_groups[2]
+
+            FROM_KEYWORD_LIST = [' from ', ' der ']
+            for from_keyword in FROM_KEYWORD_LIST:
+                from_keyword_len = len(from_keyword)
+                if certified_item.find(from_keyword) != -1:
+                    logger.warning(f'string {from_keyword} detected in certified item - shall not be here, fixing...')
+                    certified_item_first = certified_item[:certified_item.find(from_keyword)]
+                    developer = certified_item[certified_item.find(from_keyword) + from_keyword_len:]
+                    certified_item = certified_item_first
+                    continue
+
+            end_pos = developer.find('\f-')
+            if end_pos == -1:
+                end_pos = developer.find('\fBSI')
+            if end_pos == -1:
+                end_pos = developer.find('Bundesamt')
+            if end_pos != -1:
+                developer = developer[:end_pos]
+
+            items_found[constants.TAG_CERT_ID] = extract_certificates.normalize_match_string(cert_id)
+            items_found[constants.TAG_CERT_ITEM] = extract_certificates.normalize_match_string(certified_item)
+            items_found[constants.TAG_DEVELOPER] = extract_certificates.normalize_match_string(developer)
+            items_found[constants.TAG_CERT_LAB] = 'BSI'
+
+    # Process page with more detailed certificate info
+    # PP Conformance, Functionality, Assurance
+    rules_certificate_third = ['PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified']
+
+    whole_text, whole_text_with_newlines, was_unicode_decode_error = extract_certificates.load_cert_file(filepath)
+
+    for rule in rules_certificate_third:
+        rule_and_sep = rule + REGEXEC_SEP
+
+        for m in re.finditer(rule_and_sep, whole_text):
+            # check if previous rules had at least one match
+            if not constants.TAG_CERT_ID in items_found.keys():
+                logger.error('ERROR: front page not found for file: {}'.format(filepath))
+
+            match_groups = m.groups()
+            ref_protection_profiles = match_groups[0]
+            cc_version = match_groups[1]
+            cc_security_level = match_groups[2]
+
+            items_found[constants.TAG_REFERENCED_PROTECTION_PROFILES] = extract_certificates.normalize_match_string(ref_protection_profiles)
+            items_found[constants.TAG_CC_VERSION] = extract_certificates.normalize_match_string(cc_version)
+            items_found[constants.TAG_CC_SECURITY_LEVEL] = extract_certificates.normalize_match_string(cc_security_level)
+
+    # print('\n*** Certificates without detected preface:')
+    # for file_name in files_without_match:
+    #     print('No hits for {}'.format(file_name))
+    # print('Total no hits files: {}'.format(len(files_without_match)))
+    # print('\n**********************************')
+
+    return items_found
+
+def extract_keywords(filepath: Path) -> Dict[str, str]:
+    return extract_certificates.parse_cert_file(filepath, cc_search_rules, -1, extract_certificates.LINE_SEPARATOR)[0]
\ No newline at end of file
diff --git a/test/data/test_cc_oop/cc_products_active.csv b/test/data/test_cc_oop/cc_products_active.csv
index 2561a372..5b41d581 100644
--- a/test/data/test_cc_oop/cc_products_active.csv
+++ b/test/data/test_cc_oop/cc_products_active.csv
@@ -1,3 +1,3 @@
 Category,Name,Manufacturer,Scheme,Assurance Level,Protection Profile(s),Certification Date,Archived Date,Certification Report URL,Security Target URL,Maintenance Date,Maintenance Title,Maintenance Report,Maintenance ST
-Access Control Devices and Systems,NetIQ Identity Manager 4.7,NetIQ Corporation,SE,"EAL3+,ALC_FLR.2",,06/15/2020,06/15/2025,http://www.commoncriteriaportal.org:443/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf,,,,
-Access Control Devices and Systems,Magic SSO V4.0,"Dreamsecurity Co., Ltd.",KR,None,KECS-PP-0822-2017 SSO V1.0,11/15/2019,11/15/2024,http://www.commoncriteriaportal.org:443/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf,,,,
+Access Control Devices and Systems,NetIQ Identity Manager 4.7,NetIQ Corporation,SE,"EAL3+,ALC_FLR.2",,06/15/2020,06/15/2025,http://commoncriteriaportal.org:443/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf,http://commoncriteriaportal.org:443/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf,,,,
+Access Control Devices and Systems,Magic SSO V4.0,"Dreamsecurity Co., Ltd.",KR,None,KECS-PP-0822-2017 SSO V1.0,11/15/2019,11/15/2024,http://commoncriteriaportal.org:443/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf,http://commoncriteriaportal.org:443/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf,,,,
diff --git a/test/data/test_cc_oop/cc_products_active.html b/test/data/test_cc_oop/cc_products_active.html
index 463fc51a..6c44e3c0 100644
--- a/test/data/test_cc_oop/cc_products_active.html
+++ b/test/data/test_cc_oop/cc_products_active.html
@@ -149,8 +149,7 @@
                </span>

             </form>

          </div>

-         <script type="text/javascript"

-            src="//www.google.com/coop/cse/brand?form=searchbox_016233930414485990345:f_zj6spfpx4"></script>

+         <script type="text/javascript" src="//www.google.com/coop/cse/brand?form=searchbox_016233930414485990345:f_zj6spfpx4"></script>

 

          <!-- Google CSE Search Box Ends -->

 

@@ -159,8 +158,7 @@
 

          <p>

 

-            <a href="https://www.commoncriteriaportal.org/members/account/login/">Login &nbsp; <img

-                  src="/assets/images/icon_arrow.png" /> </a>

+            <a href="https://www.commoncriteriaportal.org/members/account/login/">Login &nbsp; <img src="/assets/images/icon_arrow.png" /> </a>

 

 

          </p>

@@ -197,19 +195,23 @@
                   ICCC</a></li>

             <li>

                <a href="/pps/" title="Protection Profiles">

-                  PROTECTION PROFILES </a> </li>

+                  PROTECTION PROFILES </a>

+            </li>

             <li>

                <a href="/pps/?cpp=1" title="Collaborative Protection Profiles">

-                  COLLABORATIVE PPS </a> </li>

+                  COLLABORATIVE PPS </a>

+            </li>

 

             <li> <a href="/products/" title="Certified Products">

                   CERTIFIED PRODUCTS </a></li>

             <li>

                <a href="/communities/index.cfm" title="Technology Communities">

-                  TECHNICAL COMMUNITIES</a></li>

+                  TECHNICAL COMMUNITIES</a>

+            </li>

             <li>

                <a href="/cc/">

-                  PUBLICATIONS</a> </li>

+                  PUBLICATIONS</a>

+            </li>

             <li><a href="/ccra/index.cfm">

                   ABOUT THE CC</a></li>

             <li><a href="/">HOME</a></li>

@@ -446,8 +448,7 @@
 

 

                <caption style="font-weight:bold;font-size:larger">

-                  <a name="AC" href="#AC" id="toggleAC" class="toggle"><img src="/assets/images/plusicon.gif" border="0"

-                        height="11" width="11" class="toggle" style="margin-right:.5em" />Access Control Devices and

+                  <a name="AC" href="#AC" id="toggleAC" class="toggle"><img src="/assets/images/plusicon.gif" border="0" height="11" width="11" class="toggle" style="margin-right:.5em" />Access Control Devices and

                      Systems &ndash; 27 Certified Products</a>

                </caption>

                <thead>

@@ -460,19 +461,14 @@
                   <tr class="">

                      <td class="b">

                         <!-- ID:  2020.1111 -->

-                        NetIQ Identity Manager 4.7<a name="2020.1111" style="text-decoration:none;">&nbsp;</a><span

-                           title="2020.1111">&nbsp;</span>

+                        NetIQ Identity Manager 4.7<a name="2020.1111" style="text-decoration:none;">&nbsp;</a><span title="2020.1111">&nbsp;</span>

                         <div class="none">

                            <!-- <a href="https&#x3a;&#x2f;&#x2f;www.netiq.com&#x2f;" title="Vendor's web site" target="_blank">NetIQ Corporation</a> -->

                            <img src="/assets/images/spacer.gif" width="600" height="1" class="" align="top"> <br />

                            <!-- ------ ------ ------ Document Links ------ ------ ------ -->

                            <div style="line-height:2.5em">

-                              <a href="/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf"

-                                 title="Certification Report: Certification Report - NetIQ® Identity Manager 4.7.pdf"

-                                 target="_blank" class="button2">Certification Report</a>

-                              <a href="/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf"

-                                 title="Security Target: ST - NetIQ Identity Manager 4.7.pdf" target="_blank"

-                                 class="button2">Security Target</a>

+                              <a href="/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf" title="Certification Report: Certification Report - NetIQ® Identity Manager 4.7.pdf" target="_blank" class="button2">Certification Report</a>

+                              <a href="/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf" title="Security Target: ST - NetIQ Identity Manager 4.7.pdf" target="_blank" class="button2">Security Target</a>

 

                            </div>

                            <!-- ------ ------ ------ END Document Links ------ ------ ------ -->

@@ -483,14 +479,11 @@
                      </td>

                      <!--end-product-cell-->

                      <td>

-                        <a href="https&#x3a;&#x2f;&#x2f;www.netiq.com&#x2f;" title="Vendor's web site"

-                           target="_blank">NetIQ Corporation</a>

+                        <a href="https&#x3a;&#x2f;&#x2f;www.netiq.com&#x2f;" title="Vendor's web site" target="_blank">NetIQ Corporation</a>

                      </td>

                      <td>

 

-                        <a href="/files/epfiles/Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf"

-                           title="Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf" target="_blank"

-                           class="button2">CCRA Certificate</a>

+                        <a href="/files/epfiles/Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf" title="Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf" target="_blank" class="button2">CCRA Certificate</a>

 

                      </td>

                      <td style="text-align:center"><span title="2020.1111">2020-06-15</span></td>

@@ -507,29 +500,21 @@
 

                      <td style="text-align:center">

                         <a href="/ccra/members/#SE">

-                           <img src="/assets/images/flags/se.png"

-                              alt="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC"

-                              title="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC" height="32"

-                              width="32" style="margin-bottom:-1em" /><br />SE</a>

+                           <img src="/assets/images/flags/se.png" alt="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC" title="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC" height="32" width="32" style="margin-bottom:-1em" /><br />SE</a>

                      </td>

                   </tr>

 

                   <tr class="even">

                      <td class="b">

                         <!-- ID:  2019.1265 -->

-                        Magic SSO V4.0<a name="2019.1265" style="text-decoration:none;">&nbsp;</a><span

-                           title="2019.1265">&nbsp;</span>

+                        Magic SSO V4.0<a name="2019.1265" style="text-decoration:none;">&nbsp;</a><span title="2019.1265">&nbsp;</span>

                         <div class="none">

                            <!-- <a href="https&#x3a;&#x2f;&#x2f;www.dreamsecurity.com&#x2f;" title="Vendor's web site" target="_blank">Dreamsecurity Co., Ltd.</a> -->

                            <img src="/assets/images/spacer.gif" width="600" height="1" class="" align="top"> <br />

                            <!-- ------ ------ ------ Document Links ------ ------ ------ -->

                            <div style="line-height:2.5em">

-                              <a href="/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf"

-                                 title="Certification Report: KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf"

-                                 target="_blank" class="button2">Certification Report</a>

-                              <a href="/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf"

-                                 title="Security Target: Magic_SSO_V4.0-ST-v1.4_EN.pdf" target="_blank"

-                                 class="button2">Security Target</a>

+                              <a href="/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf" title="Certification Report: KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf" target="_blank" class="button2">Certification Report</a>

+                              <a href="/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf" title="Security Target: Magic_SSO_V4.0-ST-v1.4_EN.pdf" target="_blank" class="button2">Security Target</a>

 

                               <!-- how it was before 2017.09

                            <a href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf" target="_blank" class="button2" title="PP: Korean National Protection Profile for Single Sign On V1.0">Protection Profile</a> -->

@@ -537,9 +522,7 @@
                               <!-- Set the display text for the link to the PP -->

 

                               <!-- ------------------------------------ -->

-                              <br /><a

-                                 href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf"

-                                 target="_blank" class="button2" title="">Korean National Protection Profile for Single

+                              <br /><a href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf" target="_blank" class="button2" title="">Korean National Protection Profile for Single

                                  Sign On V1.0</a>

 

                            </div>

@@ -551,8 +534,7 @@
                      </td>

                      <!--end-product-cell-->

                      <td>

-                        <a href="https&#x3a;&#x2f;&#x2f;www.dreamsecurity.com&#x2f;" title="Vendor's web site"

-                           target="_blank">Dreamsecurity Co., Ltd.</a>

+                        <a href="https&#x3a;&#x2f;&#x2f;www.dreamsecurity.com&#x2f;" title="Vendor's web site" target="_blank">Dreamsecurity Co., Ltd.</a>

                      </td>

                      <td>

 

@@ -572,10 +554,7 @@
 

                      <td style="text-align:center">

                         <a href="/ccra/members/#KR">

-                           <img src="/assets/images/flags/kr.png"

-                              alt="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;"

-                              title="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;" height="32"

-                              width="32" style="margin-bottom:-1em" /><br />KR</a>

+                           <img src="/assets/images/flags/kr.png" alt="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;" title="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;" height="32" width="32" style="margin-bottom:-1em" /><br />KR</a>

                      </td>

                   </tr>

 

diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json
index 43e8e4d1..0bde57bb 100644
--- a/test/data/test_cc_oop/fictional_cert.json
+++ b/test/data/test_cc_oop/fictional_cert.json
@@ -35,6 +35,17 @@
         "st_link_ok": true,
         "report_link_ok": true,
         "st_convert_ok": true,
-        "report_convert_ok": true
+        "report_convert_ok": true,
+        "st_extract_ok": true,
+        "report_extract_ok": true
+    },
+    "pdf_data": {
+        "_type": "PdfData",
+        "report_metadata": null,
+        "st_metadata": null,
+        "report_frontpage": null,
+        "st_frontpage": null,
+        "report_keywords": null,
+        "st_keywords": null
     }
 }
\ No newline at end of file
diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json
index ae98c9d1..bfd08c20 100644
--- a/test/data/test_cc_oop/toy_dataset.json
+++ b/test/data/test_cc_oop/toy_dataset.json
@@ -18,10 +18,10 @@
             ],
             "not_valid_before": "2020-06-15",
             "not_valid_after": "2025-06-15",
-            "report_link": "http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf",
-            "st_link": "http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf",
+            "report_link": "http://commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf",
+            "st_link": "http://commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf",
             "src": "csv + html",
-            "cert_link": "http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
+            "cert_link": "http://commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
             "manufacturer_web": "https://www.netiq.com/",
             "protection_profiles": [],
             "maintainance_updates": [],
@@ -30,7 +30,18 @@
                 "st_link_ok": true,
                 "report_link_ok": true,
                 "st_convert_ok": true,
-                "report_convert_ok": true
+                "report_convert_ok": true,
+                "st_extract_ok": true,
+                "report_extract_ok": true
+            },
+            "pdf_data": {
+                "_type": "PdfData",
+                "report_metadata": null,
+                "st_metadata": null,
+                "report_frontpage": null,
+                "st_frontpage": null,
+                "report_keywords": null,
+                "st_keywords": null
             }
         },
         {
@@ -42,8 +53,8 @@
             "security_level": [],
             "not_valid_before": "2019-11-15",
             "not_valid_after": "2024-11-15",
-            "report_link": "http://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf",
-            "st_link": "http://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf",
+            "report_link": "http://commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf",
+            "st_link": "http://commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf",
             "src": "csv + html",
             "cert_link": null,
             "manufacturer_web": "https://www.dreamsecurity.com/",
@@ -51,7 +62,7 @@
                 {
                     "_type": "ProtectionProfile",
                     "pp_name": "Korean National Protection Profile for Single Sign On V1.0",
-                    "pp_link": "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf"
+                    "pp_link": "http://commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf"
                 }
             ],
             "maintainance_updates": [],
@@ -60,7 +71,18 @@
                 "st_link_ok": true,
                 "report_link_ok": true,
                 "st_convert_ok": true,
-                "report_convert_ok": true
+                "report_convert_ok": true,
+                "st_extract_ok": true,
+                "report_extract_ok": true
+            },
+            "pdf_data": {
+                "_type": "PdfData",
+                "report_metadata": null,
+                "st_metadata": null,
+                "report_frontpage": null,
+                "st_frontpage": null,
+                "report_keywords": null,
+                "st_keywords": null
             }
         }
     ]
diff --git a/test/test_cc_oop.py b/test/test_cc_oop.py
index 22c00929..fbb64a30 100644
--- a/test/test_cc_oop.py
+++ b/test/test_cc_oop.py
@@ -23,13 +23,14 @@ class TestCommonCriteriaOOP(TestCase):
                                            'EAL3+'},
                                           date(2020, 6, 15),
                                           date(2025, 6, 15),
-                                          'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf',
-                                          'http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf',
+                                          'http://commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf',
+                                          'http://commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf',
                                           'csv + html',
-                                          'http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf',
+                                          'http://commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf',
                                           'https://www.netiq.com/',
                                           set(),
                                           set(),
+                                          None,
                                           None)
 
         self.crt_two = CommonCriteriaCert('Access Control Devices and Systems',
@@ -39,14 +40,15 @@ class TestCommonCriteriaOOP(TestCase):
                                           set(),
                                           date(2019, 11, 15),
                                           date(2024, 11, 15),
-                                          'http://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf',
-                                          'http://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf',
+                                          'http://commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf',
+                                          'http://commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf',
                                           'csv + html',
                                           None,
                                           'https://www.dreamsecurity.com/',
                                           {CommonCriteriaCert.ProtectionProfile('Korean National Protection Profile for Single Sign On V1.0',
-                                                                                'http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf')},
+                                                                                'http://commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf')},
                                           set(),
+                                          None,
                                           None)
 
         pp = CommonCriteriaCert.ProtectionProfile('sample_pp', 'http://sample.pp')
@@ -65,21 +67,22 @@ class TestCommonCriteriaOOP(TestCase):
                                                  'http://path.to/manufacturer/web',
                                                  {pp},
                                                  {update},
+                                                 None,
                                                  None)
         self.template_dataset = CCDataset({self.crt_one.dgst: self.crt_one, self.crt_two.dgst: self.crt_two}, Path('/fictional/path/to/dataset'), 'toy dataset', 'toy dataset description')
         self.template_dataset.timestamp = datetime(2020, 11, 16, hour=17, minute=4, second=14, microsecond=770153)
 
-        self.template_report_pdf_hashes = {'869415cc4b91282e': '774c41fbba980191ca40ae610b2f61484c5997417b3325b6fd68b345173bde52',
-                                          '2d010ecfb604747a': '533a5995ef8b736cc48cfda30e8aafec77d285511471e0e5a9e8007c8750203a'}
-        self.template_target_pdf_hashes = {'869415cc4b91282e': 'b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9',
-                                           '2d010ecfb604747a': '3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be'}
+        self.template_report_pdf_hashes = {'7ef8227c1aed06bb': '774c41fbba980191ca40ae610b2f61484c5997417b3325b6fd68b345173bde52',
+                                          '561a012d9a30e960': '533a5995ef8b736cc48cfda30e8aafec77d285511471e0e5a9e8007c8750203a'}
+        self.template_target_pdf_hashes = {'7ef8227c1aed06bb': 'b9a45995d9e40b2515506bbf5945e806ef021861820426c6d0a6a074090b47a9',
+                                           '561a012d9a30e960': '3c8614338899d956e9e56f1aa88d90e37df86f3310b875d9d14ec0f71e4759be'}
 
         self.template_report_txt_path = self.test_data_dir / 'report_869415cc4b91282e.txt'
         self.template_target_txt_path = self.test_data_dir / 'target_869415cc4b91282e.txt'
 
     def test_certificate_input_sanity(self):
         self.assertEqual(self.crt_one.report_link,
-                         'http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf',
+                         'http://commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ®%20Identity%20Manager%204.7.pdf',
                          'Report link contains some improperly escaped characters.')
 
     def test_download_and_convert_pdfs(self):
@@ -96,14 +99,14 @@ class TestCommonCriteriaOOP(TestCase):
             self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (certificate report) do not the template')
             self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template')
 
-            self.assertTrue(dset['869415cc4b91282e'].state.report_txt_path.exists())
-            self.assertTrue(dset['869415cc4b91282e'].state.st_txt_path.exists())
+            self.assertTrue(dset['7ef8227c1aed06bb'].state.report_txt_path.exists())
+            self.assertTrue(dset['7ef8227c1aed06bb'].state.st_txt_path.exists())
 
-            self.assertAlmostEqual(dset['869415cc4b91282e'].state.st_txt_path.stat().st_size,
+            self.assertAlmostEqual(dset['7ef8227c1aed06bb'].state.st_txt_path.stat().st_size,
                                    self.template_target_txt_path.stat().st_size,
                                    delta=1000)
 
-            self.assertAlmostEqual(dset['869415cc4b91282e'].state.report_txt_path.stat().st_size,
+            self.assertAlmostEqual(dset['7ef8227c1aed06bb'].state.report_txt_path.stat().st_size,
                                    self.template_report_txt_path.stat().st_size,
                                    delta=1000)
 
@@ -151,5 +154,6 @@ class TestCommonCriteriaOOP(TestCase):
                              'Meta files (csv, html) were not deleted properly albeit this was explicitly required.')
 
         self.assertEqual(len(dset), 2, 'The dataset should contain 2 files.')
+
         self.assertTrue(self.crt_one in dset, 'The dataset does not contain the template certificate.')
         self.assertEqual(dset, self.template_dataset, 'The loaded dataset does not match the template dataset.')