Plug-in old extractions methods to new OOP frame

- Tests for txt file processing still missing
- The internals of the txt processing methods are still authored by petrs
- They are however fully parallel and called in an OOP fashion
- Overall, all results are gathered in a single json file
- All results are implicitly paired

4 files changed, 67 insertions, 55 deletions

diff --git a/test/data/test_cc_oop/cc_products_active.csv b/test/data/test_cc_oop/cc_products_active.csv
index 2561a372..5b41d581 100644
--- a/test/data/test_cc_oop/cc_products_active.csv
+++ b/test/data/test_cc_oop/cc_products_active.csv
@@ -1,3 +1,3 @@
 Category,Name,Manufacturer,Scheme,Assurance Level,Protection Profile(s),Certification Date,Archived Date,Certification Report URL,Security Target URL,Maintenance Date,Maintenance Title,Maintenance Report,Maintenance ST
-Access Control Devices and Systems,NetIQ Identity Manager 4.7,NetIQ Corporation,SE,"EAL3+,ALC_FLR.2",,06/15/2020,06/15/2025,http://www.commoncriteriaportal.org:443/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf,,,,
-Access Control Devices and Systems,Magic SSO V4.0,"Dreamsecurity Co., Ltd.",KR,None,KECS-PP-0822-2017 SSO V1.0,11/15/2019,11/15/2024,http://www.commoncriteriaportal.org:443/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf,http://www.commoncriteriaportal.org:443/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf,,,,
+Access Control Devices and Systems,NetIQ Identity Manager 4.7,NetIQ Corporation,SE,"EAL3+,ALC_FLR.2",,06/15/2020,06/15/2025,http://commoncriteriaportal.org:443/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf,http://commoncriteriaportal.org:443/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf,,,,
+Access Control Devices and Systems,Magic SSO V4.0,"Dreamsecurity Co., Ltd.",KR,None,KECS-PP-0822-2017 SSO V1.0,11/15/2019,11/15/2024,http://commoncriteriaportal.org:443/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf,http://commoncriteriaportal.org:443/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf,,,,
diff --git a/test/data/test_cc_oop/cc_products_active.html b/test/data/test_cc_oop/cc_products_active.html
index 463fc51a..6c44e3c0 100644
--- a/test/data/test_cc_oop/cc_products_active.html
+++ b/test/data/test_cc_oop/cc_products_active.html
@@ -149,8 +149,7 @@
                </span>

             </form>

          </div>

-         <script type="text/javascript"

-            src="//www.google.com/coop/cse/brand?form=searchbox_016233930414485990345:f_zj6spfpx4"></script>

+         <script type="text/javascript" src="//www.google.com/coop/cse/brand?form=searchbox_016233930414485990345:f_zj6spfpx4"></script>

 

          <!-- Google CSE Search Box Ends -->

 

@@ -159,8 +158,7 @@
 

          <p>

 

-            <a href="https://www.commoncriteriaportal.org/members/account/login/">Login &nbsp; <img

-                  src="/assets/images/icon_arrow.png" /> </a>

+            <a href="https://www.commoncriteriaportal.org/members/account/login/">Login &nbsp; <img src="/assets/images/icon_arrow.png" /> </a>

 

 

          </p>

@@ -197,19 +195,23 @@
                   ICCC</a></li>

             <li>

                <a href="/pps/" title="Protection Profiles">

-                  PROTECTION PROFILES </a> </li>

+                  PROTECTION PROFILES </a>

+            </li>

             <li>

                <a href="/pps/?cpp=1" title="Collaborative Protection Profiles">

-                  COLLABORATIVE PPS </a> </li>

+                  COLLABORATIVE PPS </a>

+            </li>

 

             <li> <a href="/products/" title="Certified Products">

                   CERTIFIED PRODUCTS </a></li>

             <li>

                <a href="/communities/index.cfm" title="Technology Communities">

-                  TECHNICAL COMMUNITIES</a></li>

+                  TECHNICAL COMMUNITIES</a>

+            </li>

             <li>

                <a href="/cc/">

-                  PUBLICATIONS</a> </li>

+                  PUBLICATIONS</a>

+            </li>

             <li><a href="/ccra/index.cfm">

                   ABOUT THE CC</a></li>

             <li><a href="/">HOME</a></li>

@@ -446,8 +448,7 @@
 

 

                <caption style="font-weight:bold;font-size:larger">

-                  <a name="AC" href="#AC" id="toggleAC" class="toggle"><img src="/assets/images/plusicon.gif" border="0"

-                        height="11" width="11" class="toggle" style="margin-right:.5em" />Access Control Devices and

+                  <a name="AC" href="#AC" id="toggleAC" class="toggle"><img src="/assets/images/plusicon.gif" border="0" height="11" width="11" class="toggle" style="margin-right:.5em" />Access Control Devices and

                      Systems &ndash; 27 Certified Products</a>

                </caption>

                <thead>

@@ -460,19 +461,14 @@
                   <tr class="">

                      <td class="b">

                         <!-- ID:  2020.1111 -->

-                        NetIQ Identity Manager 4.7<a name="2020.1111" style="text-decoration:none;">&nbsp;</a><span

-                           title="2020.1111">&nbsp;</span>

+                        NetIQ Identity Manager 4.7<a name="2020.1111" style="text-decoration:none;">&nbsp;</a><span title="2020.1111">&nbsp;</span>

                         <div class="none">

                            <!-- <a href="https&#x3a;&#x2f;&#x2f;www.netiq.com&#x2f;" title="Vendor's web site" target="_blank">NetIQ Corporation</a> -->

                            <img src="/assets/images/spacer.gif" width="600" height="1" class="" align="top"> <br />

                            <!-- ------ ------ ------ Document Links ------ ------ ------ -->

                            <div style="line-height:2.5em">

-                              <a href="/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf"

-                                 title="Certification Report: Certification Report - NetIQ® Identity Manager 4.7.pdf"

-                                 target="_blank" class="button2">Certification Report</a>

-                              <a href="/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf"

-                                 title="Security Target: ST - NetIQ Identity Manager 4.7.pdf" target="_blank"

-                                 class="button2">Security Target</a>

+                              <a href="/files/epfiles/Certification Report - NetIQ® Identity Manager 4.7.pdf" title="Certification Report: Certification Report - NetIQ® Identity Manager 4.7.pdf" target="_blank" class="button2">Certification Report</a>

+                              <a href="/files/epfiles/ST - NetIQ Identity Manager 4.7.pdf" title="Security Target: ST - NetIQ Identity Manager 4.7.pdf" target="_blank" class="button2">Security Target</a>

 

                            </div>

                            <!-- ------ ------ ------ END Document Links ------ ------ ------ -->

@@ -483,14 +479,11 @@
                      </td>

                      <!--end-product-cell-->

                      <td>

-                        <a href="https&#x3a;&#x2f;&#x2f;www.netiq.com&#x2f;" title="Vendor's web site"

-                           target="_blank">NetIQ Corporation</a>

+                        <a href="https&#x3a;&#x2f;&#x2f;www.netiq.com&#x2f;" title="Vendor's web site" target="_blank">NetIQ Corporation</a>

                      </td>

                      <td>

 

-                        <a href="/files/epfiles/Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf"

-                           title="Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf" target="_blank"

-                           class="button2">CCRA Certificate</a>

+                        <a href="/files/epfiles/Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf" title="Certifikat CCRA - NetIQ Identity Manager 4.7_signed.pdf" target="_blank" class="button2">CCRA Certificate</a>

 

                      </td>

                      <td style="text-align:center"><span title="2020.1111">2020-06-15</span></td>

@@ -507,29 +500,21 @@
 

                      <td style="text-align:center">

                         <a href="/ccra/members/#SE">

-                           <img src="/assets/images/flags/se.png"

-                              alt="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC"

-                              title="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC" height="32"

-                              width="32" style="margin-bottom:-1em" /><br />SE</a>

+                           <img src="/assets/images/flags/se.png" alt="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC" title="SE &ndash; Swedish Certification Body for IT Security FMV/CSEC" height="32" width="32" style="margin-bottom:-1em" /><br />SE</a>

                      </td>

                   </tr>

 

                   <tr class="even">

                      <td class="b">

                         <!-- ID:  2019.1265 -->

-                        Magic SSO V4.0<a name="2019.1265" style="text-decoration:none;">&nbsp;</a><span

-                           title="2019.1265">&nbsp;</span>

+                        Magic SSO V4.0<a name="2019.1265" style="text-decoration:none;">&nbsp;</a><span title="2019.1265">&nbsp;</span>

                         <div class="none">

                            <!-- <a href="https&#x3a;&#x2f;&#x2f;www.dreamsecurity.com&#x2f;" title="Vendor's web site" target="_blank">Dreamsecurity Co., Ltd.</a> -->

                            <img src="/assets/images/spacer.gif" width="600" height="1" class="" align="top"> <br />

                            <!-- ------ ------ ------ Document Links ------ ------ ------ -->

                            <div style="line-height:2.5em">

-                              <a href="/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf"

-                                 title="Certification Report: KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf"

-                                 target="_blank" class="button2">Certification Report</a>

-                              <a href="/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf"

-                                 title="Security Target: Magic_SSO_V4.0-ST-v1.4_EN.pdf" target="_blank"

-                                 class="button2">Security Target</a>

+                              <a href="/files/epfiles/KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf" title="Certification Report: KECS-CR-19-70 Magic SSO V4.0(eng) V1.0.pdf" target="_blank" class="button2">Certification Report</a>

+                              <a href="/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf" title="Security Target: Magic_SSO_V4.0-ST-v1.4_EN.pdf" target="_blank" class="button2">Security Target</a>

 

                               <!-- how it was before 2017.09

                            <a href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf" target="_blank" class="button2" title="PP: Korean National Protection Profile for Single Sign On V1.0">Protection Profile</a> -->

@@ -537,9 +522,7 @@
                               <!-- Set the display text for the link to the PP -->

 

                               <!-- ------------------------------------ -->

-                              <br /><a

-                                 href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf"

-                                 target="_blank" class="button2" title="">Korean National Protection Profile for Single

+                              <br /><a href="/files/ppfiles/KECS-PP-0822-2017 Korean National PP for Single Sign On V1.0(eng).pdf" target="_blank" class="button2" title="">Korean National Protection Profile for Single

                                  Sign On V1.0</a>

 

                            </div>

@@ -551,8 +534,7 @@
                      </td>

                      <!--end-product-cell-->

                      <td>

-                        <a href="https&#x3a;&#x2f;&#x2f;www.dreamsecurity.com&#x2f;" title="Vendor's web site"

-                           target="_blank">Dreamsecurity Co., Ltd.</a>

+                        <a href="https&#x3a;&#x2f;&#x2f;www.dreamsecurity.com&#x2f;" title="Vendor's web site" target="_blank">Dreamsecurity Co., Ltd.</a>

                      </td>

                      <td>

 

@@ -572,10 +554,7 @@
 

                      <td style="text-align:center">

                         <a href="/ccra/members/#KR">

-                           <img src="/assets/images/flags/kr.png"

-                              alt="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;"

-                              title="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;" height="32"

-                              width="32" style="margin-bottom:-1em" /><br />KR</a>

+                           <img src="/assets/images/flags/kr.png" alt="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;" title="KR &ndash; IT Security Certification Center&#x28;ITSCC&#x29;" height="32" width="32" style="margin-bottom:-1em" /><br />KR</a>

                      </td>

                   </tr>

 

diff --git a/test/data/test_cc_oop/fictional_cert.json b/test/data/test_cc_oop/fictional_cert.json
index 43e8e4d1..0bde57bb 100644
--- a/test/data/test_cc_oop/fictional_cert.json
+++ b/test/data/test_cc_oop/fictional_cert.json
@@ -35,6 +35,17 @@
         "st_link_ok": true,
         "report_link_ok": true,
         "st_convert_ok": true,
-        "report_convert_ok": true
+        "report_convert_ok": true,
+        "st_extract_ok": true,
+        "report_extract_ok": true
+    },
+    "pdf_data": {
+        "_type": "PdfData",
+        "report_metadata": null,
+        "st_metadata": null,
+        "report_frontpage": null,
+        "st_frontpage": null,
+        "report_keywords": null,
+        "st_keywords": null
     }
 }
\ No newline at end of file
diff --git a/test/data/test_cc_oop/toy_dataset.json b/test/data/test_cc_oop/toy_dataset.json
index ae98c9d1..bfd08c20 100644
--- a/test/data/test_cc_oop/toy_dataset.json
+++ b/test/data/test_cc_oop/toy_dataset.json
@@ -18,10 +18,10 @@
             ],
             "not_valid_before": "2020-06-15",
             "not_valid_after": "2025-06-15",
-            "report_link": "http://www.commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf",
-            "st_link": "http://www.commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf",
+            "report_link": "http://commoncriteriaportal.org/files/epfiles/Certification%20Report%20-%20NetIQ\u00ae%20Identity%20Manager%204.7.pdf",
+            "st_link": "http://commoncriteriaportal.org/files/epfiles/ST%20-%20NetIQ%20Identity%20Manager%204.7.pdf",
             "src": "csv + html",
-            "cert_link": "http://www.commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
+            "cert_link": "http://commoncriteriaportal.org/files/epfiles/Certifikat%20CCRA%20-%20NetIQ%20Identity%20Manager%204.7_signed.pdf",
             "manufacturer_web": "https://www.netiq.com/",
             "protection_profiles": [],
             "maintainance_updates": [],
@@ -30,7 +30,18 @@
                 "st_link_ok": true,
                 "report_link_ok": true,
                 "st_convert_ok": true,
-                "report_convert_ok": true
+                "report_convert_ok": true,
+                "st_extract_ok": true,
+                "report_extract_ok": true
+            },
+            "pdf_data": {
+                "_type": "PdfData",
+                "report_metadata": null,
+                "st_metadata": null,
+                "report_frontpage": null,
+                "st_frontpage": null,
+                "report_keywords": null,
+                "st_keywords": null
             }
         },
         {
@@ -42,8 +53,8 @@
             "security_level": [],
             "not_valid_before": "2019-11-15",
             "not_valid_after": "2024-11-15",
-            "report_link": "http://www.commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf",
-            "st_link": "http://www.commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf",
+            "report_link": "http://commoncriteriaportal.org/files/epfiles/KECS-CR-19-70%20Magic%20SSO%20V4.0(eng)%20V1.0.pdf",
+            "st_link": "http://commoncriteriaportal.org/files/epfiles/Magic_SSO_V4.0-ST-v1.4_EN.pdf",
             "src": "csv + html",
             "cert_link": null,
             "manufacturer_web": "https://www.dreamsecurity.com/",
@@ -51,7 +62,7 @@
                 {
                     "_type": "ProtectionProfile",
                     "pp_name": "Korean National Protection Profile for Single Sign On V1.0",
-                    "pp_link": "http://www.commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf"
+                    "pp_link": "http://commoncriteriaportal.org/files/ppfiles/KECS-PP-0822-2017%20Korean%20National%20PP%20for%20Single%20Sign%20On%20V1.0(eng).pdf"
                 }
             ],
             "maintainance_updates": [],
@@ -60,7 +71,18 @@
                 "st_link_ok": true,
                 "report_link_ok": true,
                 "st_convert_ok": true,
-                "report_convert_ok": true
+                "report_convert_ok": true,
+                "st_extract_ok": true,
+                "report_extract_ok": true
+            },
+            "pdf_data": {
+                "_type": "PdfData",
+                "report_metadata": null,
+                "st_metadata": null,
+                "report_frontpage": null,
+                "st_frontpage": null,
+                "report_keywords": null,
+                "st_keywords": null
             }
         }
     ]