From 734cbe49675183feab7f18fb083e33a3f1d826e1 Mon Sep 17 00:00:00 2001 From: J08nY Date: Mon, 3 Oct 2022 16:15:39 +0200 Subject: Add final run notebook. --- notebooks/final_run.ipynb | 201 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 notebooks/final_run.ipynb diff --git a/notebooks/final_run.ipynb b/notebooks/final_run.ipynb new file mode 100644 index 00000000..6a1b817b --- /dev/null +++ b/notebooks/final_run.ipynb @@ -0,0 +1,201 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sec_certs.dataset import CCDataset\n", + "from pathlib import Path\n", + "import shutil\n", + "import subprocess" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# A directory with a dump of the documents that will enrich the dataset.\n", + "doc_dump_dir = Path(\"cc_certs_03_10_2022\")\n", + "\n", + "# An output directory for the dataset.\n", + "dataset_dir = Path(\"cc_03_10_2022\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Print tool versions\n", + "subprocess.run([\"pdftotext\", \"-v\"])\n", + "subprocess.run([\"tesseract\", \"-v\"]);" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Create the dataset\n", + "dset = CCDataset(root_dir=dataset_dir, name=\"cc\", description=\"Final run on 03.10.2022\")" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Get the metadata\n", + "dset.get_certs_from_web()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Enrich the dataset with the pre-downloaded PDFs (collected over multiple runs to fill in the timeouts and disappearances).\n", + "for cert in dset:\n", + " report_pdf = doc_dump_dir / \"report\" / \"pdf\" / f\"{cert.dgst}.pdf\"\n", + " if report_pdf.exists():\n", + " shutil.copy(report_pdf, cert.state.report_pdf_path)\n", + " cert.state.report_download_ok = True\n", + " target_pdf = doc_dump_dir / \"target\" / \"pdf\" / f\"{cert.dgst}.pdf\"\n", + " if target_pdf.exists():\n", + " shutil.copy(target_pdf, cert.state.target_pdf_path)\n", + " cert.state.target_download_ok = True" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Try to download last missing (or ones not present in the pre-downloaded)\n", + "dset.download_all_pdfs(fresh=False)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Convert all\n", + "dset.convert_all_pdfs()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Analyze all\n", + "dset.analyze_certificates()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# And do maintenance updates as well\n", + "dset.process_maintenance_updates()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "# Finally, dump it all\n", + "dset.to_json()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file -- cgit v1.3.1 From e2829a09bb667b855997e8e8966fc8e4f17a22dd Mon Sep 17 00:00:00 2001 From: J08nY Date: Mon, 3 Oct 2022 19:30:34 +0200 Subject: Add new CC cert state attributes. --- sec_certs/sample/common_criteria.py | 43 +++++++++++++++++++++--------- tests/data/test_cc_oop/fictional_cert.json | 14 +++++----- tests/data/test_cc_oop/toy_dataset.json | 28 ++++++++++--------- 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py index 4bdd44ab..a9298eab 100644 --- a/sec_certs/sample/common_criteria.py +++ b/sec_certs/sample/common_criteria.py @@ -104,16 +104,18 @@ class CommonCriteriaCert( @dataclass(init=False) class InternalState(ComplexSerializableType): """ - Holds internal state of the dataset, whether downloads and converts of individual components succeeded. Also + Holds internal state of the certificate, whether downloads and converts of individual components succeeded. Also holds information about errors and paths to the files. """ - st_download_ok: bool - report_download_ok: bool - st_convert_ok: bool - report_convert_ok: bool - st_extract_ok: bool - report_extract_ok: bool + st_download_ok: bool # Whether target download went OK + report_download_ok: bool # Whether report download went OK + st_convert_garbage: bool # Whether initial target conversion resulted in garbage + report_convert_garbage: bool # Whether initial report conversion resulted in garbage + st_convert_ok: bool # Whether overall target conversion went OK (either pdftotext or via OCR) + report_convert_ok: bool # Whether overall report conversion went OK (either pdftotext or via OCR) + st_extract_ok: bool # Whether target extraction went OK + report_extract_ok: bool # Whether report extraction went OK errors: List[str] @@ -129,20 +131,25 @@ class CommonCriteriaCert( def __init__( self, - st_download_ok: bool = True, - report_download_ok: bool = True, - st_convert_ok: bool = True, - report_convert_ok: bool = True, - st_extract_ok: bool = True, - report_extract_ok: bool = True, + st_download_ok: bool = False, + report_download_ok: bool = False, + st_convert_garbage: bool = False, + report_convert_garbage: bool = False, + st_convert_ok: bool = False, + report_convert_ok: bool = False, + st_extract_ok: bool = False, + report_extract_ok: bool = False, errors: Optional[List[str]] = None, st_pdf_hash: Optional[str] = None, report_pdf_hash: Optional[str] = None, st_txt_hash: Optional[str] = None, report_txt_hash: Optional[str] = None, ): + super().__init__() self.st_download_ok = st_download_ok self.report_download_ok = report_download_ok + self.st_convert_garbage = st_convert_garbage + self.report_convert_garbage = report_convert_garbage self.st_convert_ok = st_convert_ok self.report_convert_ok = report_convert_ok self.st_extract_ok = st_extract_ok @@ -158,6 +165,8 @@ class CommonCriteriaCert( return [ "st_download_ok", "report_download_ok", + "st_convert_garbage", + "report_convert_garbage", "st_convert_ok", "report_convert_ok", "st_extract_ok", @@ -797,6 +806,7 @@ class CommonCriteriaCert( cert.state.report_download_ok = False cert.state.errors.append(error_msg) else: + cert.state.report_download_ok = True cert.state.report_pdf_hash = helpers.get_sha256_filepath(cert.state.report_pdf_path) cert.pdf_data.report_filename = unquote_plus(str(urlparse(cert.report_link).path).split("/")[-1]) return cert @@ -820,6 +830,7 @@ class CommonCriteriaCert( cert.state.st_download_ok = False cert.state.errors.append(error_msg) else: + cert.state.st_download_ok = True cert.state.st_pdf_hash = helpers.get_sha256_filepath(cert.state.st_pdf_path) cert.pdf_data.st_filename = unquote_plus(str(urlparse(cert.st_link).path).split("/")[-1]) return cert @@ -839,6 +850,7 @@ class CommonCriteriaCert( cert.state.report_convert_ok = False cert.state.errors.append(error_msg) else: + cert.state.report_convert_ok = True cert.state.report_txt_hash = helpers.get_sha256_filepath(cert.state.report_txt_path) return cert @@ -857,6 +869,7 @@ class CommonCriteriaCert( cert.state.st_convert_ok = False cert.state.errors.append(error_msg) else: + cert.state.st_convert_ok = True cert.state.st_txt_hash = helpers.get_sha256_filepath(cert.state.st_txt_path) return cert @@ -872,6 +885,8 @@ class CommonCriteriaCert( if response != constants.RETURNCODE_OK: cert.state.st_extract_ok = False cert.state.errors.append(response) + else: + cert.state.st_extract_ok = True return cert @staticmethod @@ -886,6 +901,8 @@ class CommonCriteriaCert( if response != constants.RETURNCODE_OK: cert.state.report_extract_ok = False cert.state.errors.append(response) + else: + cert.state.report_extract_ok = True return cert @staticmethod diff --git a/tests/data/test_cc_oop/fictional_cert.json b/tests/data/test_cc_oop/fictional_cert.json index 2b52af75..117f87d2 100644 --- a/tests/data/test_cc_oop/fictional_cert.json +++ b/tests/data/test_cc_oop/fictional_cert.json @@ -41,12 +41,14 @@ }, "state": { "_type": "sec_certs.sample.common_criteria.CommonCriteriaCert.InternalState", - "st_download_ok": true, - "report_download_ok": true, - "st_convert_ok": true, - "report_convert_ok": true, - "st_extract_ok": true, - "report_extract_ok": true, + "st_download_ok": false, + "report_download_ok": false, + "st_convert_garbage": false, + "report_convert_garbage": false, + "st_convert_ok": false, + "report_convert_ok": false, + "st_extract_ok": false, + "report_extract_ok": false, "errors": [], "st_pdf_hash": null, "report_pdf_hash": null, diff --git a/tests/data/test_cc_oop/toy_dataset.json b/tests/data/test_cc_oop/toy_dataset.json index 19f22405..cb733b0c 100644 --- a/tests/data/test_cc_oop/toy_dataset.json +++ b/tests/data/test_cc_oop/toy_dataset.json @@ -44,12 +44,14 @@ }, "state": { "_type": "sec_certs.sample.common_criteria.CommonCriteriaCert.InternalState", - "st_download_ok": true, - "report_download_ok": true, - "st_convert_ok": true, - "report_convert_ok": true, - "st_extract_ok": true, - "report_extract_ok": true, + "st_download_ok": false, + "report_download_ok": false, + "st_convert_garbage": false, + "report_convert_garbage": false, + "st_convert_ok": false, + "report_convert_ok": false, + "st_extract_ok": false, + "report_extract_ok": false, "errors": [], "st_pdf_hash": null, "report_pdf_hash": null, @@ -130,12 +132,14 @@ }, "state": { "_type": "sec_certs.sample.common_criteria.CommonCriteriaCert.InternalState", - "st_download_ok": true, - "report_download_ok": true, - "st_convert_ok": true, - "report_convert_ok": true, - "st_extract_ok": true, - "report_extract_ok": true, + "st_download_ok": false, + "report_download_ok": false, + "st_convert_garbage": false, + "report_convert_garbage": false, + "st_convert_ok": false, + "report_convert_ok": false, + "st_extract_ok": false, + "report_extract_ok": false, "errors": [], "st_pdf_hash": null, "report_pdf_hash": null, -- cgit v1.3.1 From 78d961a8ec2bfc3fa134918672d8100beaf18d05 Mon Sep 17 00:00:00 2001 From: J08nY Date: Mon, 3 Oct 2022 19:53:04 +0200 Subject: Store OCR results into CC cert state. --- sec_certs/sample/common_criteria.py | 22 ++++++++++++++-------- sec_certs/sample/fips.py | 4 ++-- sec_certs/utils/pdf.py | 19 +++++++++++-------- tests/test_cc_oop.py | 5 +++++ 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py index a9298eab..b3ccb8db 100644 --- a/sec_certs/sample/common_criteria.py +++ b/sec_certs/sample/common_criteria.py @@ -843,14 +843,18 @@ class CommonCriteriaCert( :param CommonCriteriaCert cert: cert to download the pdf report for :return CommonCriteriaCert: the modified certificate with updated state """ - exit_code = sec_certs.utils.pdf.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path) - if exit_code != constants.RETURNCODE_OK: + ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file( + cert.state.report_pdf_path, cert.state.report_txt_path + ) + # If OCR was done the result was garbage + cert.state.report_convert_garbage = ocr_done + # And put the whole result into convert_ok + cert.state.report_convert_ok = ok_result + if not ok_result: error_msg = "failed to convert report pdf->txt" logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - cert.state.report_convert_ok = False cert.state.errors.append(error_msg) else: - cert.state.report_convert_ok = True cert.state.report_txt_hash = helpers.get_sha256_filepath(cert.state.report_txt_path) return cert @@ -862,14 +866,16 @@ class CommonCriteriaCert( :param CommonCriteriaCert cert: cert to download the pdf security target for :return CommonCriteriaCert: the modified certificate with updated state """ - exit_code = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path) - if exit_code != constants.RETURNCODE_OK: + ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path) + # If OCR was done the result was garbage + cert.state.st_convert_garbage = ocr_done + # And put the whole result into convert_ok + cert.state.st_convert_ok = ok_result + if not ok_result: error_msg = "failed to convert security target pdf->txt" logger.error(f"Cert dgst: {cert.dgst} " + error_msg) - cert.state.st_convert_ok = False cert.state.errors.append(error_msg) else: - cert.state.st_convert_ok = True cert.state.st_txt_hash = helpers.get_sha256_filepath(cert.state.st_txt_path) return cert diff --git a/sec_certs/sample/fips.py b/sec_certs/sample/fips.py index cf45aafb..eb8308f5 100644 --- a/sec_certs/sample/fips.py +++ b/sec_certs/sample/fips.py @@ -559,8 +559,8 @@ class FIPSCertificate(Certificate["FIPSCertificate", "FIPSCertificate.Heuristics """ cert, pdf_path, txt_path = tup if not cert.state.txt_state: - exit_code = sec_certs.utils.pdf.convert_pdf_file(pdf_path, txt_path) - if exit_code != constants.RETURNCODE_OK: + ocr_done, ok_result = sec_certs.utils.pdf.convert_pdf_file(pdf_path, txt_path) + if not ok_result: logger.error(f"Cert dgst: {cert.cert_id} failed to convert security policy pdf->txt") cert.state.txt_state = False else: diff --git a/sec_certs/utils/pdf.py b/sec_certs/utils/pdf.py index 08de2cc7..a40fb18d 100644 --- a/sec_certs/utils/pdf.py +++ b/sec_certs/utils/pdf.py @@ -64,15 +64,18 @@ def ocr_pdf_file(pdf_path: Path) -> str: return contents -def convert_pdf_file(pdf_path: Path, txt_path: Path) -> str: +def convert_pdf_file(pdf_path: Path, txt_path: Path) -> Tuple[bool, bool]: """ Convert a PDF tile to text and save it on the `txt_path`. :param pdf_path: Path to the to-be-converted PDF file. :param txt_path: Path to the resulting text file. - :return: Whether the conversion was successful (see constants). + :return: A tuple of two results, whether OCR was done and what the complete result + was (OK/NOK). """ txt = None + ok = False + ocr = False try: with pdf_path.open("rb") as pdf_handle: pdf = pdftotext.PDF(pdf_handle, "", True) # No password, Raw=True @@ -82,19 +85,19 @@ def convert_pdf_file(pdf_path: Path, txt_path: Path) -> str: if txt is None or text_is_garbage(txt): logger.warning(f"Detected garbage during conversion of {pdf_path}") + ocr = True try: txt = ocr_pdf_file(pdf_path) logger.info(f"OCR OK for {pdf_path}") except Exception as e: logger.error(f"Error during OCR of {pdf_path}, using garbage: {e}") - if txt is None: - return constants.RETURNCODE_NOK + if txt is not None: + ok = True + with txt_path.open("w", encoding="utf-8") as txt_handle: + txt_handle.write(txt) - with txt_path.open("w", encoding="utf-8") as txt_handle: - txt_handle.write(txt) - - return constants.RETURNCODE_OK + return ocr, ok def parse_pdf_date(dateval: Optional[bytes]) -> Optional[datetime]: diff --git a/tests/test_cc_oop.py b/tests/test_cc_oop.py index 201b3db3..0777a1e1 100644 --- a/tests/test_cc_oop.py +++ b/tests/test_cc_oop.py @@ -142,6 +142,11 @@ class TestCommonCriteriaOOP(TestCase): f"Hash of PDF security target for CommonCriteriaCert with digest {cert.dgst} deviates from template.", ) + self.assertFalse(dset["309ac2fd7f2dcf17"].state.report_convert_garbage) + self.assertFalse(dset["309ac2fd7f2dcf17"].state.st_convert_garbage) + self.assertTrue(dset["309ac2fd7f2dcf17"].state.report_convert_ok) + self.assertTrue(dset["309ac2fd7f2dcf17"].state.st_convert_ok) + self.assertTrue(dset["309ac2fd7f2dcf17"].state.report_txt_path.exists()) self.assertTrue(dset["309ac2fd7f2dcf17"].state.st_txt_path.exists()) -- cgit v1.3.1 From 83c30715731ba39091f6c7cb1f4d6c6d1590731e Mon Sep 17 00:00:00 2001 From: J08nY Date: Mon, 3 Oct 2022 22:32:47 +0200 Subject: Fix extraction check with new attr defaults. --- sec_certs/sample/common_criteria.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py index b3ccb8db..3a9dcc1f 100644 --- a/sec_certs/sample/common_criteria.py +++ b/sec_certs/sample/common_criteria.py @@ -192,13 +192,13 @@ class CommonCriteriaCert( def report_is_ok_to_analyze(self, fresh: bool = True) -> bool: if fresh is True: - return self.report_download_ok and self.report_convert_ok and self.report_extract_ok + return self.report_download_ok and self.report_convert_ok else: return self.report_download_ok and self.report_convert_ok and not self.report_extract_ok def st_is_ok_to_analyze(self, fresh: bool = True) -> bool: if fresh is True: - return self.st_download_ok and self.st_convert_ok and self.st_extract_ok + return self.st_download_ok and self.st_convert_ok else: return self.st_download_ok and self.st_convert_ok and not self.st_extract_ok -- cgit v1.3.1 From 9763c1d8a61b75112efddbbe61a4642e1e4b46ce Mon Sep 17 00:00:00 2001 From: J08nY Date: Tue, 4 Oct 2022 14:40:33 +0200 Subject: Add final-run logs. --- notebooks/final_run.ipynb | 789 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 700 insertions(+), 89 deletions(-) diff --git a/notebooks/final_run.ipynb b/notebooks/final_run.ipynb index 6a1b817b..8b982523 100644 --- a/notebooks/final_run.ipynb +++ b/notebooks/final_run.ipynb @@ -2,10 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, + "execution_count": 1, + "metadata": {}, "outputs": [], "source": [ "from sec_certs.dataset import CCDataset\n", @@ -16,74 +14,159 @@ }, { "cell_type": "code", - "execution_count": null, - "outputs": [], + "execution_count": 2, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n", + "True\n" + ] + } + ], "source": [ "# A directory with a dump of the documents that will enrich the dataset.\n", "doc_dump_dir = Path(\"cc_certs_03_10_2022\")\n", + "print(doc_dump_dir.exists())\n", "\n", "# An output directory for the dataset.\n", - "dataset_dir = Path(\"cc_03_10_2022\")" - ], + "dataset_dir = Path(\"cc_03_10_2022\")\n", + "print(dataset_dir.exists())" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "metadata": { - "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tesseract 5.2.0\n", + " leptonica-1.82.0\n", + " libgif 5.2.1 : libjpeg 8d (libjpeg-turbo 2.1.3) : libpng 1.6.38 : libtiff 4.4.0 : zlib 1.2.12 : libwebp 1.2.4 : libopenjp2 2.5.0\n", + " Found AVX2\n", + " Found AVX\n", + " Found FMA\n", + " Found SSE4.1\n", + " Found OpenMP 201511\n", + " Found libarchive 3.6.1 zlib/1.2.12 liblzma/5.2.5 bz2lib/1.0.8 liblz4/1.9.3 libzstd/1.5.2\n", + " Found libcurl/7.85.0 OpenSSL/1.1.1q zlib/1.2.12 brotli/1.0.9 zstd/1.5.2 libidn2/2.3.3 libpsl/0.21.1 (+libidn2/2.3.0) libssh2/1.10.0 nghttp2/1.50.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "pdftotext version 22.09.0\n", + "Copyright 2005-2022 The Poppler Developers - http://poppler.freedesktop.org\n", + "Copyright 1996-2011, 2022 Glyph & Cog, LLC\n" + ] + } + ], "source": [ "# Print tool versions\n", "subprocess.run([\"pdftotext\", \"-v\"])\n", "subprocess.run([\"tesseract\", \"-v\"]);" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": { - "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": null, + }, "outputs": [], "source": [ "# Create the dataset\n", "dset = CCDataset(root_dir=dataset_dir, name=\"cc\", description=\"Final run on 03.10.2022\")" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": { - "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.01s/it]\n", + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:02<00:00, 1.00s/it]\n", + "The CSV cc_03_10_2022/web/cc_products_active.csv contains 8 duplicates by the primary key.\n", + "The CSV cc_03_10_2022/web/cc_products_archived.csv contains 10 duplicates by the primary key.\n", + "When merging certificates with dgst b26ce64e0c677e3d, the following mismatch occured: Attribute=security_level, self[security_level]={'ALC_DVS.1', 'EAL2+'}, other[security_level]={'EAL2'}\n", + "When merging certificates with dgst 7f043162f88a1c3a, the following mismatch occured: Attribute=not_valid_after, self[not_valid_after]=2024-09-19, other[not_valid_after]=2024-09-18\n", + "When merging certificates with dgst 109eb2158ca6a2f9, the following mismatch occured: Attribute=security_level, self[security_level]={'ADV_IMP.2', 'EAL4+', 'AVA_VLA.4', 'ATE_DPT.2', 'AVA_MSU.3'}, other[security_level]={'ADV_DVS.2', 'AVA_MSU.3', 'EAL5+', 'AVA_VLA.4'}\n", + "When merging certificates with dgst c437d26a62a22c39, the following mismatch occured: Attribute=not_valid_after, self[not_valid_after]=2019-11-19, other[not_valid_after]=2018-05-07\n" + ] + } + ], "source": [ "# Get the metadata\n", "dset.get_certs_from_web()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Duplicate entry in PP dataset: ('Protection Profile for Enterprise Security Management Policy Management', 'https://www.commoncriteriaportal.org/files/ppfiles/PP_ESM_PM_V1.4.pdf')\n", + "Duplicate entry in PP dataset: ('Stateful Traffic Filter Firewall Extended Package for Network Device Protection Profile', 'https://www.commoncriteriaportal.org/files/ppfiles/PP_ND_TFFWEP_V1.0.pdf')\n", + "Duplicate entry in PP dataset: ('Stateful Traffic Filter Firewall Extended Package for Network Device Protection Profile', 'https://www.commoncriteriaportal.org/files/ppfiles/PP_ND_TFFWEP_V1.0.pdf')\n", + "Duplicate entry in PP dataset: ('Public Key-Enabled Application Family of Protection Profiles', 'https://www.commoncriteriaportal.org/files/ppfiles/PP_VID3004-PP.pdf')\n", + "Duplicate entry in PP dataset: ('Protection Profile for Wireless Local Area Network (WLAN) Access Systems', 'https://www.commoncriteriaportal.org/files/ppfiles/PP_WLAN_AS_V1.0.pdf')\n", + "Duplicate entry in PP dataset: ('PC Client Specific Trusted Platform Module Family 1.2; Level 2, Version 1.1', 'https://www.commoncriteriaportal.org/files/ppfiles/pp0030b.pdf')\n", + "Duplicate entry in PP dataset: ('Protection Profile for Enterprise Security Management Access Control', 'https://www.commoncriteriaportal.org/files/ppfiles/pp_esm_ac_v2.1.pdf')\n", + "Duplicate entry in PP dataset: ('U.S. Government Protection Profile Intrusion Detection System - System for Basic Robustness Environments, Version 1.7', 'https://www.commoncriteriaportal.org/files/ppfiles/pp_ids_sys_br_v1.7.pdf')\n", + "Duplicate entry in PP dataset: ('Network Device Protection Profile Extended Package SIP Server', 'https://www.commoncriteriaportal.org/files/ppfiles/pp_nd_sip_ep_v1.1.pdf')\n", + "Duplicate entry in PP dataset: ('Network Device Protection Profile Extended Package VPN Gateway', 'https://www.commoncriteriaportal.org/files/ppfiles/pp_nd_vpn_gw_ep_v1.1.pdf')\n", + "Duplicate entry in PP dataset: ('Extended Package for Secure Shell (SSH), Version 1.0', 'https://www.commoncriteriaportal.org/files/ppfiles/pp_ssh_ep_v1.0.pdf')\n", + "Duplicate entry in PP dataset: ('Protection Profile for Software Full Disk Encryption', 'https://www.commoncriteriaportal.org/files/ppfiles/pp_swfde_v1.1.pdf')\n", + "Duplicate entry in PP dataset: ('Smart Card Security User Group - Smart Card Protection Profile, Version 3.0', 'https://www.commoncriteriaportal.org/files/ppfiles/scsugpp.pdf')\n" + ] + } ], + "source": [ + "# Process the profiles?\n", + "dset.process_protection_profiles()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": { - "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": null, + }, "outputs": [], "source": [ "# Enrich the dataset with the pre-downloaded PDFs (collected over multiple runs to fill in the timeouts and disappearances).\n", + "dset.reports_pdf_dir.mkdir(parents=True, exist_ok=True)\n", + "dset.targets_pdf_dir.mkdir(parents=True, exist_ok=True)\n", "for cert in dset:\n", " report_pdf = doc_dump_dir / \"report\" / \"pdf\" / f\"{cert.dgst}.pdf\"\n", " if report_pdf.exists():\n", @@ -91,111 +174,639 @@ " cert.state.report_download_ok = True\n", " target_pdf = doc_dump_dir / \"target\" / \"pdf\" / f\"{cert.dgst}.pdf\"\n", " if target_pdf.exists():\n", - " shutil.copy(target_pdf, cert.state.target_pdf_path)\n", - " cert.state.target_download_ok = True" - ], + " shutil.copy(target_pdf, cert.state.st_pdf_path)\n", + " cert.state.st_download_ok = True" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": { - "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Downloading reports: 0%| | 0/15 [00:00txt: poppler error creating document\n", + "Converting reports to txt: 2%|██ | 115/5112 [00:08<06:00, 13.86it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: bfc86f14e2484335 failed to convert report pdf->txt\n", + "Converting reports to txt: 4%|███▏ | 181/5112 [00:11<04:31, 18.14it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/1ee7ecee9e7e131c.pdf\n", + "Converting reports to txt: 4%|███▍ | 194/5112 [00:12<05:14, 15.65it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d8c205b4924f91b3.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/20874d4fa6e4c878.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/f8a79eb20ade29af.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3477723044183b31.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/82c24f729c2e0092.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b3757b6bb5f884d4.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/0f9468a2e263d3a3.pdf\n", + "Converting reports to txt: 4%|███▋ | 206/5112 [00:27<05:55, 13.79it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/66bbd8fb83f0abbb.pdf\n", + "Converting reports to txt: 4%|███▌ | 207/5112 [00:48<1:18:01, 1.05it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/e1daa354ae5a61fd.pdf\n", + "Converting reports to txt: 4%|███▌ | 208/5112 [04:37<12:22:36, 9.09s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7a0e44767b51f0c4.pdf\n", + "Converting reports to txt: 4%|███▌ | 210/5112 [04:50<11:56:08, 8.77s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/051066781c36ad55.pdf\n", + "Converting reports to txt: 4%|███▋ | 211/5112 [05:08<12:55:52, 9.50s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/03bce905b71945aa.pdf\n", + "Converting reports to txt: 4%|███▋ | 212/5112 [05:09<11:42:39, 8.60s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/0286c223fc4e626a.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/559e4549a852aab2.pdf\n", + "Converting reports to txt: 4%|███▊ | 216/5112 [05:19<7:48:52, 5.75s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c5e25f90c7006546.pdf\n", + "Converting reports to txt: 4%|███▊ | 218/5112 [06:39<24:17:02, 17.86s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/744a7a202d909323.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/449c74a92ebb61a4.pdf\n", + "Converting reports to txt: 4%|███▋ | 220/5112 [41:28<455:51:29, 335.46s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c80801f9a71b030e.pdf\n", + "Converting reports to txt: 6%|█████▊ | 332/5112 [41:37<2:39:36, 2.00s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/852cad2f53bda148.pdf\n", + "Converting reports to txt: 7%|██████ | 347/5112 [41:38<1:49:37, 1.38s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b0deb8e9d0026b64.pdf\n", + "Converting reports to txt: 7%|██████ | 348/5112 [41:53<2:21:55, 1.79s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/56be1d745bbb4994.pdf\n", + "Converting reports to txt: 9%|███████▊ | 438/5112 [42:04<14:48, 5.26it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/af6767c887ae930a.pdf\n", + "Converting reports to txt: 9%|████████ | 453/5112 [42:22<11:43, 6.62it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d2f75cdaeb322e7f.pdf\n", + "Converting reports to txt: 9%|███████▉ | 454/5112 [42:34<1:17:44, 1.00s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d7300e1bfafed26e.pdf\n", + "Converting reports to txt: 9%|███████▉ | 457/5112 [44:00<5:15:52, 4.07s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7015af58766db091.pdf\n", + "Converting reports to txt: 9%|███████▉ | 463/5112 [45:54<10:28:38, 8.11s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/85328590c0d7273f.pdf\n", + "Converting reports to txt: 9%|████████▏ | 472/5112 [45:59<5:49:23, 4.52s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/e56e14e8cbf89e3c.pdf\n", + "Converting reports to txt: 9%|████████▎ | 474/5112 [46:03<5:22:17, 4.17s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4844355021ea22f1.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/21a1deda31c3e213.pdf\n", + "Converting reports to txt: 9%|████████▎ | 475/5112 [46:09<5:35:16, 4.34s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d8aff63d56506605.pdf\n", + "Converting reports to txt: 9%|████████▏ | 476/5112 [46:47<10:25:46, 8.10s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/aca070da96add3b6.pdf\n", + "Converting reports to txt: 9%|████████▍ | 485/5112 [47:21<6:41:42, 5.21s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c8982f2e8de39b22.pdf\n", + "Converting reports to txt: 10%|████████▍ | 487/5112 [47:22<5:36:04, 4.36s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/977b04d04def60a7.pdf\n", + "Converting reports to txt: 10%|████████▌ | 492/5112 [47:56<6:50:08, 5.33s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6eeed6fefb1f0243.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/62313881916550ca.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6f8d7a6a1dea6a3a.pdf\n", + "Converting reports to txt: 10%|████████▌ | 495/5112 [49:13<13:25:37, 10.47s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/e6cffd14e732a030.pdf\n", + "Converting reports to txt: 10%|████████▌ | 499/5112 [50:08<15:02:09, 11.73s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/47913a485c3c8a18.pdf\n", + "Converting reports to txt: 10%|████████▌ | 501/5112 [50:09<11:42:57, 9.15s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b0b7b073ca2dfe5f.pdf\n", + "Converting reports to txt: 10%|████████▋ | 502/5112 [50:14<10:47:14, 8.42s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/60f3bd10ee9be85b.pdf\n", + "Converting reports to txt: 10%|████████▋ | 503/5112 [50:54<17:36:40, 13.76s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/5c3806bf79eeab7f.pdf\n", + "Converting reports to txt: 10%|████████▊ | 511/5112 [52:08<13:44:26, 10.75s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4c9468f20fdb04f7.pdf\n", + "Converting reports to txt: 10%|████████▉ | 516/5112 [52:09<8:18:32, 6.51s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/9ee9f3c598318ae5.pdf\n", + "Converting reports to txt: 11%|█████████▍ | 543/5112 [52:29<1:56:00, 1.52s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/afde7a688dcc5c83.pdf\n", + "Converting reports to txt: 17%|███████████████▏ | 850/5112 [53:26<07:47, 9.11it/s]Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 98668cafb8dda26f failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 02958808b57334f0 failed to convert report pdf->txt\n", + "Cert dgst: 4de4a9f436958574 failed to convert report pdf->txt\n", + "Converting reports to txt: 22%|████████████████████ | 1139/5112 [53:48<04:11, 15.78it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/9903c89cbb35eec1.pdf\n", + "Converting reports to txt: 24%|█████████████████████▋ | 1234/5112 [53:55<04:44, 13.62it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/1faf7bf1f74a1851.pdf\n", + "Converting reports to txt: 28%|████████████████████████▊ | 1409/5112 [54:08<04:12, 14.66it/s]Error when converting pdf->txt: poppler error creating document\n", + "Converting reports to txt: 28%|█████████████████████████ | 1427/5112 [54:09<04:11, 14.66it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 422dc5758723c7d1 failed to convert report pdf->txt\n", + "Converting reports to txt: 32%|████████████████████████████▍ | 1615/5112 [54:22<03:35, 16.21it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b93dfa0a8ec3609c.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6ed5861434d5fed9.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/209e2d18e1fa0cca.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3df01446a2b92093.pdf\n", + "Converting reports to txt: 35%|███████████████████████████████▏ | 1770/5112 [54:38<06:44, 8.26it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3e5591d086a54905.pdf\n", + "Converting reports to txt: 36%|████████████████████████████████▌ | 1852/5112 [54:44<04:35, 11.85it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d3568613c552f9e8.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/d3568613c552f9e8.pdf, using garbage: pdftoppm failed: 99\n", + "Converting reports to txt: 38%|██████████████████████████████████▏ | 1941/5112 [54:53<04:42, 11.24it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/822e5f25e7c4631e.pdf\n", + "Converting reports to txt: 38%|██████████████████████████████████▌ | 1965/5112 [55:00<11:07, 4.71it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 2d05e976e1adf696 failed to convert report pdf->txt\n", + "Converting reports to txt: 41%|█████████████████████████████████████▏ | 2112/5112 [55:09<03:26, 14.54it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/69d373903e9635ef.pdf\n", + "Converting reports to txt: 42%|█████████████████████████████████████▊ | 2146/5112 [55:13<04:25, 11.18it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/ae1b80e9cccec491.pdf\n", + "Converting reports to txt: 43%|██████████████████████████████████████▎ | 2178/5112 [55:25<11:34, 4.22it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/21d5f1b05e1ba491.pdf\n", + "Converting reports to txt: 47%|█████████████████████████████████████████▉ | 2383/5112 [55:53<03:52, 11.72it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3f8cee6d5bee5397.pdf\n", + "Converting reports to txt: 56%|██████████████████████████████████████████████████▋ | 2879/5112 [56:42<04:16, 8.72it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7e32023021d5aad2.pdf\n", + "Converting reports to txt: 59%|████████████████████████████████████████████████████▉ | 3010/5112 [56:58<03:20, 10.50it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b43100f226a61f53.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/b43100f226a61f53.pdf, using garbage: pdftoppm failed: 99\n", + "Converting reports to txt: 75%|███████████████████████████████████████████████████████████████████▌ | 3836/5112 [58:04<01:24, 15.13it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6db03eff148934a4.pdf\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/03cc150ce40cf816.pdf\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error during OCR of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: f9c8da9deff77ab5 failed to convert report pdf->txt\n", + "Converting reports to txt: 76%|████████████████████████████████████████████████████████████████████▌ | 3896/5112 [58:08<01:13, 16.65it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3570791ff9c92912.pdf\n", + "Converting reports to txt: 77%|█████████████████████████████████████████████████████████████████████▍ | 3947/5112 [58:11<01:21, 14.22it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/ed886279d0d61096.pdf\n", + "Converting reports to txt: 86%|█████████████████████████████████████████████████████████████████████████████▍ | 4399/5112 [58:56<00:59, 11.91it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c763b37bb95517e9.pdf\n", + "Converting reports to txt: 86%|█████████████████████████████████████████████████████████████████████████████▋ | 4416/5112 [58:57<00:54, 12.77it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/40640508ac3a37f2.pdf\n", + "Converting reports to txt: 87%|██████████████████████████████████████████████████████████████████████████████ | 4433/5112 [58:59<00:50, 13.40it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 7147c2f70d983d57 failed to convert report pdf->txt\n", + "Converting reports to txt: 95%|█████████████████████████████████████████████████████████████████████████████████████▎ | 4843/5112 [59:31<00:22, 12.22it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3ad7749ef678ce95.pdf\n", + "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████ | 4888/5112 [59:34<00:14, 15.80it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/1683817e5db17d9c.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b762fcb2983aa148.pdf\n", + "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████▍ | 4911/5112 [59:36<00:15, 12.93it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/cc681ca95621a6db.pdf\n", + "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████▋ | 4924/5112 [59:37<00:14, 12.83it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/889693d2b11bdfaf.pdf\n", + "Converting reports to txt: 97%|██████████████████████████████████████████████████████████████████████████████████████▉ | 4941/5112 [59:38<00:13, 12.37it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/876cb2fe02017f3f.pdf\n", + "Converting reports to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 5112/5112 [59:57<00:00, 1.42it/s]\n", + "Converting targets to txt: 2%|██ | 116/5034 [00:26<18:20, 4.47it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf\n", + "Error during OCR of cc_03_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: bfc86f14e2484335 failed to convert security target pdf->txt\n", + "Converting targets to txt: 10%|█████████▎ | 518/5034 [02:36<21:29, 3.50it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf\n", + "Converting targets to txt: 10%|█████████▍ | 525/5034 [02:41<32:08, 2.34it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 4c9468f20fdb04f7 failed to convert security target pdf->txt\n", + "Converting targets to txt: 21%|███████████████████ | 1066/5034 [05:00<10:51, 6.09it/s]Error when converting pdf->txt: poppler error creating document\n", + "Converting targets to txt: 21%|███████████████████▏ | 1074/5034 [05:01<10:50, 6.08it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf\n", + "Converting targets to txt: 21%|███████████████████▎ | 1081/5034 [05:02<10:34, 6.23it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: a2b962c7f1d2bc56 failed to convert security target pdf->txt\n", + "Converting targets to txt: 28%|█████████████████████████▎ | 1414/5034 [06:30<13:27, 4.48it/s]Error when converting pdf->txt: poppler error creating document\n", + "Converting targets to txt: 28%|█████████████████████████▍ | 1422/5034 [06:33<17:06, 3.52it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf\n", + "Converting targets to txt: 28%|█████████████████████████▌ | 1429/5034 [06:35<15:29, 3.88it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 422dc5758723c7d1 failed to convert security target pdf->txt\n", + "Converting targets to txt: 31%|████████████████████████████▎ | 1583/5034 [07:15<16:44, 3.44it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/063d3805a3b12930.pdf\n", + "Converting targets to txt: 33%|█████████████████████████████▌ | 1653/5034 [07:31<11:11, 5.03it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/2f4a41e8a5dfac6a.pdf\n", + "Converting targets to txt: 33%|██████████████████████████████ | 1684/5034 [07:36<10:06, 5.52it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/504258938266b87f.pdf\n", + "Converting targets to txt: 36%|███████████████████████████████▉ | 1789/5034 [08:01<11:47, 4.59it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/de85ff4c31069f12.pdf\n", + "Converting targets to txt: 37%|█████████████████████████████████ | 1851/5034 [08:16<12:16, 4.32it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/d3568613c552f9e8.pdf\n", + "Converting targets to txt: 37%|█████████████████████████████████▏ | 1855/5034 [08:17<12:45, 4.15it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/d3568613c552f9e8.pdf, using garbage: pdftoppm failed: 99\n", + "Converting targets to txt: 40%|████████████████████████████████████ | 2018/5034 [08:56<13:00, 3.86it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/b4bc79d8a558c7cf.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/09b43147f1307352.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/ce987167d42db722.pdf\n", + "Converting targets to txt: 40%|████████████████████████████████████▏ | 2022/5034 [08:58<13:34, 3.70it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/c16a92b40550193d.pdf\n", + "Converting targets to txt: 40%|███████████████████████████████████▌ | 2034/5034 [10:13<2:34:56, 3.10s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/2ff761edd4ed9b72.pdf\n", + "Converting targets to txt: 41%|███████████████████████████████████▉ | 2054/5034 [12:03<2:20:59, 2.84s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/9e4d3347efd95ec9.pdf\n", + "Converting targets to txt: 43%|███████████████████████████████████████ | 2185/5034 [12:42<13:25, 3.54it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/ad9ea5a4acf9ac53.pdf\n", + "Converting targets to txt: 44%|██████████████████████████████████████▎ | 2194/5034 [14:51<4:20:08, 5.50s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/28922ef97b2dfaa0.pdf\n", + "Converting targets to txt: 44%|█████████████████████████████████████▉ | 2196/5034 [17:07<17:33:53, 22.28s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/5c715225c805354f.pdf\n", + "Converting targets to txt: 68%|█████████████████████████████████████████████████████████████▌ | 3446/5034 [23:37<06:33, 4.04it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/731102b849e8d104.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/f65cab49ea5422dd.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/863d91c6a2b8e77a.pdf\n", + "Converting targets to txt: 77%|█████████████████████████████████████████████████████████████████████ | 3862/5034 [25:32<04:09, 4.69it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/18e5179ba124af1d.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/bc31fd0273ab72f8.pdf\n", + "Converting targets to txt: 79%|██████████████████████████████████████████████████████████████████████▉ | 3971/5034 [26:03<05:13, 3.39it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/5dfeb0219325989b.pdf\n", + "Converting targets to txt: 79%|███████████████████████████████████████████████████████████████████████ | 3975/5034 [26:05<05:21, 3.29it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/22617e916bad6f53.pdf\n", + "Converting targets to txt: 79%|███████████████████████████████████████████████████████████████████████ | 3978/5034 [26:06<05:33, 3.17it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/6344672e3178f63e.pdf\n", + "Converting targets to txt: 80%|███████████████████████████████████████████████████████████████████████▊ | 4014/5034 [32:17<44:48, 2.64s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/483dc36fcf11b257.pdf\n", + "Converting targets to txt: 80%|██████████████████████████████████████████████████████████████████████▎ | 4024/5034 [33:13<1:15:50, 4.51s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/5885f28f042038c1.pdf\n", + "Converting targets to txt: 80%|████████████████████████████████████████████████████████████████████████▎ | 4048/5034 [33:36<16:35, 1.01s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/7b08ad74a882337c.pdf\n", + "Converting targets to txt: 87%|██████████████████████████████████████████████████████████████████████████████▎ | 4380/5034 [36:12<02:11, 4.97it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf\n", + "Error during OCR of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 7147c2f70d983d57 failed to convert security target pdf->txt\n", + "Converting targets to txt: 90%|████████████████████████████████████████████████████████████████████████████████▉ | 4525/5034 [36:56<02:32, 3.33it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/931e1b2a68a1bcd2.pdf\n", + "Converting targets to txt: 92%|██████████████████████████████████████████████████████████████████████████████████▌ | 4615/5034 [37:15<01:37, 4.31it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/16229c97ce083e8c.pdf\n", + "Converting targets to txt: 99%|████████████████████████████████████████████████████████████████████████████████████████▉ | 4977/5034 [38:46<00:13, 4.22it/s]Error when converting pdf->txt: poppler error creating page\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/a552244ec3ca8f60.pdf\n", + "Converting targets to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 5034/5034 [40:44<00:00, 2.06it/s]\n", + "Converting reports to txt: 0%| | 0/8 [00:00txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 02958808b57334f0 failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: bfc86f14e2484335 failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: f9c8da9deff77ab5 failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 422dc5758723c7d1 failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 7147c2f70d983d57 failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 98668cafb8dda26f failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 2d05e976e1adf696 failed to convert report pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 4de4a9f436958574 failed to convert report pdf->txt\n", + "Converting reports to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:01<00:00, 7.98it/s]\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Converting targets to txt: 0%| | 0/5 [00:00txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf\n", + "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf\n", + "Error during OCR of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: a2b962c7f1d2bc56 failed to convert security target pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 7147c2f70d983d57 failed to convert security target pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: bfc86f14e2484335 failed to convert security target pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 422dc5758723c7d1 failed to convert security target pdf->txt\n", + "Error during OCR of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 4c9468f20fdb04f7 failed to convert security target pdf->txt\n", + "Converting targets to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:01<00:00, 4.99it/s]\n" + ] } - } - }, - { - "cell_type": "code", - "execution_count": null, - "outputs": [], + ], "source": [ "# Convert all\n", "dset.convert_all_pdfs()" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": { - "collapsed": false, "pycharm": { "name": "#%%\n" + }, + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Extracting report metadata: 0%| | 0/5104 [00:00txt: poppler error creating document\n", + "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf\n", + "ERROR:sec_certs.utils.pdf:Error during OCR of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf, using garbage: pdftoppm failed: 1\n", + "ERROR:sec_certs.sample.certificate:Cert dgst: cert_8ba5d4d02ea73d3d_update_d879046e9be61396 failed to convert report pdf->txt\n", + "Converting reports to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 457/457 [00:21<00:00, 21.21it/s]\n", + "Converting targets to txt: 36%|█████████████████████████████████████████████████████ | 163/448 [00:42<00:42, 6.71it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/targets/pdf/cert_09b43147f1307352_update_a1ff8f3d472941f7.pdf\n", + "Converting targets to txt: 74%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 333/448 [01:21<00:20, 5.51it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/targets/pdf/cert_bd5c7bb29151de01_update_070f8e523f3deba1.pdf\n", + "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/targets/pdf/cert_ef9565620c399b7e_update_b169fd5fee2eb72b.pdf\n", + "Converting targets to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 448/448 [04:58<00:00, 1.50it/s]\n", + "ERROR:sec_certs.utils.pdf:Error when converting pdf->txt: poppler error creating document\n", + "Converting reports to txt: 0%| | 0/1 [00:00txt\n", + "Converting reports to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.00s/it]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Extracting report metadata: 0%| | 0/456 [00:00" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# And do maintenance updates as well\n", "dset.process_maintenance_updates()" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": { - "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "# Finally, dump it all\n", + "dset.to_json()" + ] }, { "cell_type": "code", "execution_count": null, + "metadata": {}, "outputs": [], - "source": [ - "# Finally, dump it all\n", - "dset.to_json()" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.10.7" } }, "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file + "nbformat_minor": 1 +} -- cgit v1.3.1