diff options
| author | Adam Janovsky | 2022-10-04 20:25:18 +0200 |
|---|---|---|
| committer | Adam Janovsky | 2022-10-04 20:25:18 +0200 |
| commit | 56e7ed87bee0fafde4254bdbcea589b0e81592ab (patch) | |
| tree | f06b79291ce8b25ef683cf6d0012463502463065 | |
| parent | 8b6322cabd53f54998af2fc65ef4e428cffa7960 (diff) | |
| parent | ac9767b5a8687af1dcc116016e1a26c944891602 (diff) | |
| download | sec-certs-56e7ed87bee0fafde4254bdbcea589b0e81592ab.tar.gz sec-certs-56e7ed87bee0fafde4254bdbcea589b0e81592ab.tar.zst sec-certs-56e7ed87bee0fafde4254bdbcea589b0e81592ab.zip | |
Merge branch 'main' into feat/eval-cpe-matches
| -rw-r--r-- | notebooks/final_run.ipynb | 664 | ||||
| -rw-r--r-- | sec_certs/sample/common_criteria.py | 4 |
2 files changed, 334 insertions, 334 deletions
diff --git a/notebooks/final_run.ipynb b/notebooks/final_run.ipynb index 8b982523..9e9a8b90 100644 --- a/notebooks/final_run.ipynb +++ b/notebooks/final_run.ipynb @@ -26,17 +26,17 @@ "output_type": "stream", "text": [ "True\n", - "True\n" + "False\n" ] } ], "source": [ "# A directory with a dump of the documents that will enrich the dataset.\n", - "doc_dump_dir = Path(\"cc_certs_03_10_2022\")\n", + "doc_dump_dir = Path(\"cc_certs_04_10_2022\")\n", "print(doc_dump_dir.exists())\n", "\n", "# An output directory for the dataset.\n", - "dataset_dir = Path(\"cc_03_10_2022\")\n", + "dataset_dir = Path(\"cc_04_10_2022\")\n", "print(dataset_dir.exists())" ] }, @@ -92,7 +92,7 @@ "outputs": [], "source": [ "# Create the dataset\n", - "dset = CCDataset(root_dir=dataset_dir, name=\"cc\", description=\"Final run on 03.10.2022\")" + "dset = CCDataset(root_dir=dataset_dir, name=\"cc\", description=\"Final run on 04.10.2022\")" ] }, { @@ -108,13 +108,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.01s/it]\n", + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.00s/it]\n", "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:02<00:00, 1.00s/it]\n", - "The CSV cc_03_10_2022/web/cc_products_active.csv contains 8 duplicates by the primary key.\n", - "The CSV cc_03_10_2022/web/cc_products_archived.csv contains 10 duplicates by the primary key.\n", + "The CSV cc_04_10_2022/web/cc_products_active.csv contains 8 duplicates by the primary key.\n", + "The CSV cc_04_10_2022/web/cc_products_archived.csv contains 10 duplicates by the primary key.\n", "When merging certificates with dgst b26ce64e0c677e3d, the following mismatch occured: Attribute=security_level, self[security_level]={'ALC_DVS.1', 'EAL2+'}, other[security_level]={'EAL2'}\n", "When merging certificates with dgst 7f043162f88a1c3a, the following mismatch occured: Attribute=not_valid_after, self[not_valid_after]=2024-09-19, other[not_valid_after]=2024-09-18\n", - "When merging certificates with dgst 109eb2158ca6a2f9, the following mismatch occured: Attribute=security_level, self[security_level]={'ADV_IMP.2', 'EAL4+', 'AVA_VLA.4', 'ATE_DPT.2', 'AVA_MSU.3'}, other[security_level]={'ADV_DVS.2', 'AVA_MSU.3', 'EAL5+', 'AVA_VLA.4'}\n", + "When merging certificates with dgst 109eb2158ca6a2f9, the following mismatch occured: Attribute=security_level, self[security_level]={'EAL4+', 'AVA_MSU.3', 'ATE_DPT.2', 'ADV_IMP.2', 'AVA_VLA.4'}, other[security_level]={'ADV_DVS.2', 'EAL5+', 'AVA_MSU.3', 'AVA_VLA.4'}\n", "When merging certificates with dgst c437d26a62a22c39, the following mismatch occured: Attribute=not_valid_after, self[not_valid_after]=2019-11-19, other[not_valid_after]=2018-05-07\n" ] } @@ -126,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -156,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": { "pycharm": { "name": "#%%\n" @@ -180,7 +180,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": { "pycharm": { "name": "#%%\n" @@ -191,24 +191,24 @@ "name": "stderr", "output_type": "stream", "text": [ - "Downloading reports: 0%| | 0/15 [00:00<?, ?it/s]Cert dgst: e27eb9efbd67c4f7 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/Document%20SAFER%20Blue%202%20Security%20Target%20V1.10_???.pdf, code: nok\n", - "Cert dgst: 8ac22970bd8042d2 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid4016b-vr.pdf, code: nok\n", - "Cert dgst: 82fcce2db9ef6063 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/emCA%20Certification%20Report%20v1.0.pdf, code: nok\n", + "Downloading reports: 0%| | 0/20 [00:00<?, ?it/s]Cert dgst: 82fcce2db9ef6063 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/emCA%20Certification%20Report%20v1.0.pdf, code: nok\n", + "Cert dgst: e27eb9efbd67c4f7 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/Document%20SAFER%20Blue%202%20Security%20Target%20V1.10_???.pdf, code: nok\n", + "Downloading reports: 10%|██████████ | 2/20 [00:01<00:09, 2.00it/s]Cert dgst: 305af0eec85bb856 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/[KECS-CR-13-28]%20AhnLab%20TrusGuard%20V2.2(??).pdf, code: nok\n", "Cert dgst: d79ecebfe21f2144 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid10024a-vr.pdf, code: nok\n", - "Cert dgst: 305af0eec85bb856 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/[KECS-CR-13-28]%20AhnLab%20TrusGuard%20V2.2(??).pdf, code: nok\n", - "Cert dgst: a90d78924e91dc6c failed to download report from https://www.commoncriteriaportal.org/files/epfiles/ANSSI-CC_2010-03en1.pdf, code: nok\n", + "Cert dgst: 8ac22970bd8042d2 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid4016b-vr.pdf, code: nok\n", "Cert dgst: 02482228eb547c15 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid4016a-vr.pdf, code: nok\n", - "Downloading reports: 47%|███████████████████████████████████████████████▏ | 7/15 [00:01<00:01, 6.99it/s]Cert dgst: ace7ea6d7f58dbb1 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/0308a_pdf.pdf, code: nok\n", - "Cert dgst: 391c8b9254ed6ed9 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/silverpeak-v31-cert-eng.pdf, code: nok\n", + "Cert dgst: a90d78924e91dc6c failed to download report from https://www.commoncriteriaportal.org/files/epfiles/ANSSI-CC_2010-03en1.pdf, code: nok\n", + "Cert dgst: ace7ea6d7f58dbb1 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/0308a_pdf.pdf, code: nok\n", "Cert dgst: 811ca609aebe2e96 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid10355-vr.pdf, code: nok\n", + "Cert dgst: 391c8b9254ed6ed9 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/silverpeak-v31-cert-eng.pdf, code: nok\n", "Cert dgst: d0d0dfe4860db6be failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid1023-vr.pdf, code: nok\n", - "Cert dgst: 6af4a27251001715 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid10491-vr, code: nok\n", "Cert dgst: 7d074a56ee50aec4 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid3012a-vr.pdf, code: nok\n", + "Downloading reports: 90%|██████████████████████████████████████████████████████████████████████████████████████████ | 18/20 [00:02<00:00, 10.22it/s]Cert dgst: 6af4a27251001715 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/st_vid10491-vr, code: nok\n", "Cert dgst: 1a8ac8fbe5bfcda9 failed to download report from https://www.commoncriteriaportal.org/files/epfiles/SERTIT-017_CR_Thinklogical_VX160_KVM_Matrix_switch_v1.01.pdf, code: nok\n", - "Downloading reports: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 15/15 [00:03<00:00, 4.99it/s]\n", + "Downloading reports: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 20/20 [00:03<00:00, 6.65it/s]\n", "Downloading targets: 0%| | 0/14 [00:00<?, ?it/s]Cert dgst: e27eb9efbd67c4f7 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/Document%20SAFER%20Blue%202%20Security%20Target%20V1.10_???.pdf, code: nok\n", - "Cert dgst: 305af0eec85bb856 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/CC-TG22-???????_??.pdf, code: nok\n", "Cert dgst: a90d78924e91dc6c failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/ANSSI-CC-cible_2010-03en1.pdf, code: nok\n", + "Cert dgst: 305af0eec85bb856 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/CC-TG22-???????_??.pdf, code: nok\n", "Cert dgst: 02482228eb547c15 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/st_vid4016a-st.pdf, code: nok\n", "Downloading targets: 29%|████████████████████████████▊ | 4/14 [00:01<00:02, 3.99it/s]Cert dgst: 811ca609aebe2e96 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/st_vid10355-st.pdf, code: nok\n", "Cert dgst: 391c8b9254ed6ed9 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/silverpeak-v31-sec-eng.pdf, code: nok\n", @@ -225,7 +225,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": { "pycharm": { "name": "#%%\n" @@ -237,228 +237,240 @@ "name": "stderr", "output_type": "stream", "text": [ - "Converting reports to txt: 2%|█▋ | 95/5112 [00:07<06:21, 13.17it/s]Error when converting pdf->txt: poppler error creating document\n", - "Converting reports to txt: 2%|██ | 115/5112 [00:08<06:00, 13.86it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Converting reports to txt: 2%|█▋ | 96/5115 [00:07<06:29, 12.89it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf\n", + "Converting reports to txt: 2%|██ | 114/5115 [00:09<06:14, 13.36it/s]Error during OCR of cc_04_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: bfc86f14e2484335 failed to convert report pdf->txt\n", - "Converting reports to txt: 4%|███▏ | 181/5112 [00:11<04:31, 18.14it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/1ee7ecee9e7e131c.pdf\n", - "Converting reports to txt: 4%|███▍ | 194/5112 [00:12<05:14, 15.65it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d8c205b4924f91b3.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/20874d4fa6e4c878.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/f8a79eb20ade29af.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3477723044183b31.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/82c24f729c2e0092.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b3757b6bb5f884d4.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/0f9468a2e263d3a3.pdf\n", - "Converting reports to txt: 4%|███▋ | 206/5112 [00:27<05:55, 13.79it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/66bbd8fb83f0abbb.pdf\n", - "Converting reports to txt: 4%|███▌ | 207/5112 [00:48<1:18:01, 1.05it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/e1daa354ae5a61fd.pdf\n", - "Converting reports to txt: 4%|███▌ | 208/5112 [04:37<12:22:36, 9.09s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7a0e44767b51f0c4.pdf\n", - "Converting reports to txt: 4%|███▌ | 210/5112 [04:50<11:56:08, 8.77s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/051066781c36ad55.pdf\n", - "Converting reports to txt: 4%|███▋ | 211/5112 [05:08<12:55:52, 9.50s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/03bce905b71945aa.pdf\n", - "Converting reports to txt: 4%|███▋ | 212/5112 [05:09<11:42:39, 8.60s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/0286c223fc4e626a.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/559e4549a852aab2.pdf\n", - "Converting reports to txt: 4%|███▊ | 216/5112 [05:19<7:48:52, 5.75s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c5e25f90c7006546.pdf\n", - "Converting reports to txt: 4%|███▊ | 218/5112 [06:39<24:17:02, 17.86s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/744a7a202d909323.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/449c74a92ebb61a4.pdf\n", - "Converting reports to txt: 4%|███▋ | 220/5112 [41:28<455:51:29, 335.46s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c80801f9a71b030e.pdf\n", - "Converting reports to txt: 6%|█████▊ | 332/5112 [41:37<2:39:36, 2.00s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/852cad2f53bda148.pdf\n", - "Converting reports to txt: 7%|██████ | 347/5112 [41:38<1:49:37, 1.38s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b0deb8e9d0026b64.pdf\n", - "Converting reports to txt: 7%|██████ | 348/5112 [41:53<2:21:55, 1.79s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/56be1d745bbb4994.pdf\n", - "Converting reports to txt: 9%|███████▊ | 438/5112 [42:04<14:48, 5.26it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/af6767c887ae930a.pdf\n", - "Converting reports to txt: 9%|████████ | 453/5112 [42:22<11:43, 6.62it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d2f75cdaeb322e7f.pdf\n", - "Converting reports to txt: 9%|███████▉ | 454/5112 [42:34<1:17:44, 1.00s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d7300e1bfafed26e.pdf\n", - "Converting reports to txt: 9%|███████▉ | 457/5112 [44:00<5:15:52, 4.07s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7015af58766db091.pdf\n", - "Converting reports to txt: 9%|███████▉ | 463/5112 [45:54<10:28:38, 8.11s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/85328590c0d7273f.pdf\n", - "Converting reports to txt: 9%|████████▏ | 472/5112 [45:59<5:49:23, 4.52s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/e56e14e8cbf89e3c.pdf\n", - "Converting reports to txt: 9%|████████▎ | 474/5112 [46:03<5:22:17, 4.17s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4844355021ea22f1.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/21a1deda31c3e213.pdf\n", - "Converting reports to txt: 9%|████████▎ | 475/5112 [46:09<5:35:16, 4.34s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d8aff63d56506605.pdf\n", - "Converting reports to txt: 9%|████████▏ | 476/5112 [46:47<10:25:46, 8.10s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/aca070da96add3b6.pdf\n", - "Converting reports to txt: 9%|████████▍ | 485/5112 [47:21<6:41:42, 5.21s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c8982f2e8de39b22.pdf\n", - "Converting reports to txt: 10%|████████▍ | 487/5112 [47:22<5:36:04, 4.36s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/977b04d04def60a7.pdf\n", - "Converting reports to txt: 10%|████████▌ | 492/5112 [47:56<6:50:08, 5.33s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6eeed6fefb1f0243.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/62313881916550ca.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6f8d7a6a1dea6a3a.pdf\n", - "Converting reports to txt: 10%|████████▌ | 495/5112 [49:13<13:25:37, 10.47s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/e6cffd14e732a030.pdf\n", - "Converting reports to txt: 10%|████████▌ | 499/5112 [50:08<15:02:09, 11.73s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/47913a485c3c8a18.pdf\n", - "Converting reports to txt: 10%|████████▌ | 501/5112 [50:09<11:42:57, 9.15s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b0b7b073ca2dfe5f.pdf\n", - "Converting reports to txt: 10%|████████▋ | 502/5112 [50:14<10:47:14, 8.42s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/60f3bd10ee9be85b.pdf\n", - "Converting reports to txt: 10%|████████▋ | 503/5112 [50:54<17:36:40, 13.76s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/5c3806bf79eeab7f.pdf\n", - "Converting reports to txt: 10%|████████▊ | 511/5112 [52:08<13:44:26, 10.75s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4c9468f20fdb04f7.pdf\n", - "Converting reports to txt: 10%|████████▉ | 516/5112 [52:09<8:18:32, 6.51s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/9ee9f3c598318ae5.pdf\n", - "Converting reports to txt: 11%|█████████▍ | 543/5112 [52:29<1:56:00, 1.52s/it]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/afde7a688dcc5c83.pdf\n", - "Converting reports to txt: 17%|███████████████▏ | 850/5112 [53:26<07:47, 9.11it/s]Error when converting pdf->txt: poppler error creating document\n", + "Converting reports to txt: 3%|███ | 173/5115 [00:12<05:17, 15.55it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/1ee7ecee9e7e131c.pdf\n", + "Converting reports to txt: 4%|███▎ | 189/5115 [00:13<05:16, 15.54it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/d8c205b4924f91b3.pdf\n", + "Converting reports to txt: 4%|███▋ | 204/5115 [00:14<05:30, 14.87it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/20874d4fa6e4c878.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/3477723044183b31.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/f8a79eb20ade29af.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/0f9468a2e263d3a3.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/82c24f729c2e0092.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/b3757b6bb5f884d4.pdf\n", + "Converting reports to txt: 4%|███▋ | 209/5115 [00:29<07:04, 11.57it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/66bbd8fb83f0abbb.pdf\n", + "Converting reports to txt: 4%|███▋ | 210/5115 [00:35<58:42, 1.39it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/e1daa354ae5a61fd.pdf\n", + "Converting reports to txt: 4%|███▋ | 211/5115 [02:52<8:57:46, 6.58s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/7a0e44767b51f0c4.pdf\n", + "Converting reports to txt: 4%|███▋ | 213/5115 [03:10<9:28:06, 6.95s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/051066781c36ad55.pdf\n", + "Converting reports to txt: 4%|███▋ | 214/5115 [03:11<8:41:14, 6.38s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/03bce905b71945aa.pdf\n", + "Converting reports to txt: 4%|███▋ | 215/5115 [03:14<8:02:35, 5.91s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/0286c223fc4e626a.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/559e4549a852aab2.pdf\n", + "Converting reports to txt: 4%|███▊ | 220/5115 [03:40<7:24:20, 5.45s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/c5e25f90c7006546.pdf\n", + "Converting reports to txt: 4%|███▊ | 221/5115 [04:38<17:34:45, 12.93s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/744a7a202d909323.pdf\n", + "Converting reports to txt: 4%|███▊ | 222/5115 [06:18<37:23:01, 27.50s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/449c74a92ebb61a4.pdf\n", + "Converting reports to txt: 4%|███▊ | 223/5115 [06:26<32:20:55, 23.81s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/c80801f9a71b030e.pdf\n", + "Converting reports to txt: 7%|██████▏ | 345/5115 [06:38<11:53, 6.69it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/852cad2f53bda148.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/b0deb8e9d0026b64.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/56be1d745bbb4994.pdf\n", + "Converting reports to txt: 9%|███████▉ | 448/5115 [06:55<07:16, 10.69it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/af6767c887ae930a.pdf\n", + "Converting reports to txt: 9%|████████ | 455/5115 [07:09<07:56, 9.77it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/d2f75cdaeb322e7f.pdf\n", + "Converting reports to txt: 9%|███████▉ | 457/5115 [08:37<5:32:23, 4.28s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/d7300e1bfafed26e.pdf\n", + "Converting reports to txt: 9%|████████ | 460/5115 [09:06<7:22:02, 5.70s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/7015af58766db091.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/85328590c0d7273f.pdf\n", + "Converting reports to txt: 9%|████████▏ | 467/5115 [10:09<9:08:17, 7.08s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/e56e14e8cbf89e3c.pdf\n", + "Converting reports to txt: 9%|████████▏ | 474/5115 [10:18<6:16:36, 4.87s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/4844355021ea22f1.pdf\n", + "Converting reports to txt: 9%|████████▎ | 476/5115 [10:21<5:41:24, 4.42s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/21a1deda31c3e213.pdf\n", + "Converting reports to txt: 9%|████████▎ | 477/5115 [10:28<5:59:05, 4.65s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/d8aff63d56506605.pdf\n", + "Converting reports to txt: 9%|████████▎ | 478/5115 [10:32<5:53:35, 4.58s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/aca070da96add3b6.pdf\n", + "Converting reports to txt: 9%|████████▍ | 483/5115 [11:23<8:43:35, 6.78s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/c8982f2e8de39b22.pdf\n", + "Converting reports to txt: 10%|████████▌ | 490/5115 [12:03<9:20:15, 7.27s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/977b04d04def60a7.pdf\n", + "Converting reports to txt: 10%|████████▌ | 494/5115 [12:04<5:58:51, 4.66s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/6eeed6fefb1f0243.pdf\n", + "Converting reports to txt: 10%|████████▌ | 496/5115 [13:26<15:06:59, 11.78s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/62313881916550ca.pdf\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Converting reports to txt: 10%|████████▌ | 497/5115 [13:38<15:10:43, 11.83s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/6f8d7a6a1dea6a3a.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/e6cffd14e732a030.pdf\n", + "Converting reports to txt: 10%|████████▌ | 498/5115 [13:57<16:59:25, 13.25s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/47913a485c3c8a18.pdf\n", + "Converting reports to txt: 10%|████████▊ | 503/5115 [14:01<7:40:59, 6.00s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/b0b7b073ca2dfe5f.pdf\n", + "Converting reports to txt: 10%|████████▋ | 504/5115 [14:27<11:21:27, 8.87s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/60f3bd10ee9be85b.pdf\n", + "Converting reports to txt: 10%|████████▋ | 505/5115 [15:07<18:08:17, 14.16s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/5c3806bf79eeab7f.pdf\n", + "Converting reports to txt: 10%|████████▊ | 515/5115 [16:20<11:26:38, 8.96s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/4c9468f20fdb04f7.pdf\n", + "Converting reports to txt: 10%|█████████ | 518/5115 [16:21<8:38:28, 6.77s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/9ee9f3c598318ae5.pdf\n", + "Converting reports to txt: 11%|█████████▍ | 542/5115 [17:08<2:22:59, 1.88s/it]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/afde7a688dcc5c83.pdf\n", + "Converting reports to txt: 17%|███████████████ | 850/5115 [18:10<09:48, 7.24it/s]Error when converting pdf->txt: poppler error creating document\n", "Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/4de4a9f436958574.pdf\n", "Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf, using garbage: pdftoppm failed: 1\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/02958808b57334f0.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 98668cafb8dda26f failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf, using garbage: pdftoppm failed: 1\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/02958808b57334f0.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/4de4a9f436958574.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 02958808b57334f0 failed to convert report pdf->txt\n", "Cert dgst: 4de4a9f436958574 failed to convert report pdf->txt\n", - "Converting reports to txt: 22%|████████████████████ | 1139/5112 [53:48<04:11, 15.78it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/9903c89cbb35eec1.pdf\n", - "Converting reports to txt: 24%|█████████████████████▋ | 1234/5112 [53:55<04:44, 13.62it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/1faf7bf1f74a1851.pdf\n", - "Converting reports to txt: 28%|████████████████████████▊ | 1409/5112 [54:08<04:12, 14.66it/s]Error when converting pdf->txt: poppler error creating document\n", - "Converting reports to txt: 28%|█████████████████████████ | 1427/5112 [54:09<04:11, 14.66it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Converting reports to txt: 22%|███████████████████▉ | 1133/5115 [18:33<04:55, 13.49it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/9903c89cbb35eec1.pdf\n", + "Converting reports to txt: 24%|█████████████████████▊ | 1238/5115 [18:41<05:01, 12.85it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/1faf7bf1f74a1851.pdf\n", + "Converting reports to txt: 28%|█████████████████████████ | 1422/5115 [18:56<04:48, 12.81it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 422dc5758723c7d1 failed to convert report pdf->txt\n", - "Converting reports to txt: 32%|████████████████████████████▍ | 1615/5112 [54:22<03:35, 16.21it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b93dfa0a8ec3609c.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6ed5861434d5fed9.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/209e2d18e1fa0cca.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3df01446a2b92093.pdf\n", - "Converting reports to txt: 35%|███████████████████████████████▏ | 1770/5112 [54:38<06:44, 8.26it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3e5591d086a54905.pdf\n", - "Converting reports to txt: 36%|████████████████████████████████▌ | 1852/5112 [54:44<04:35, 11.85it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/d3568613c552f9e8.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/d3568613c552f9e8.pdf, using garbage: pdftoppm failed: 99\n", - "Converting reports to txt: 38%|██████████████████████████████████▏ | 1941/5112 [54:53<04:42, 11.24it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/822e5f25e7c4631e.pdf\n", - "Converting reports to txt: 38%|██████████████████████████████████▌ | 1965/5112 [55:00<11:07, 4.71it/s]Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf, using garbage: pdftoppm failed: 1\n", + "Converting reports to txt: 31%|████████████████████████████▏ | 1604/5115 [19:10<04:21, 13.41it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/209e2d18e1fa0cca.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/b93dfa0a8ec3609c.pdf\n", + "Converting reports to txt: 32%|████████████████████████████▌ | 1624/5115 [19:11<03:53, 14.93it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/6ed5861434d5fed9.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/3df01446a2b92093.pdf\n", + "Converting reports to txt: 35%|███████████████████████████████▏ | 1773/5115 [19:28<06:59, 7.96it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/3e5591d086a54905.pdf\n", + "Converting reports to txt: 36%|████████████████████████████████▋ | 1859/5115 [19:35<04:54, 11.07it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/d3568613c552f9e8.pdf\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/d3568613c552f9e8.pdf, using garbage: pdftoppm failed: 99\n", + "Converting reports to txt: 38%|██████████████████████████████████▏ | 1945/5115 [19:44<04:56, 10.69it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/822e5f25e7c4631e.pdf\n", + "Converting reports to txt: 39%|██████████████████████████████████▋ | 1973/5115 [19:49<07:26, 7.04it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 2d05e976e1adf696 failed to convert report pdf->txt\n", - "Converting reports to txt: 41%|█████████████████████████████████████▏ | 2112/5112 [55:09<03:26, 14.54it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/69d373903e9635ef.pdf\n", - "Converting reports to txt: 42%|█████████████████████████████████████▊ | 2146/5112 [55:13<04:25, 11.18it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/ae1b80e9cccec491.pdf\n", - "Converting reports to txt: 43%|██████████████████████████████████████▎ | 2178/5112 [55:25<11:34, 4.22it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/21d5f1b05e1ba491.pdf\n", - "Converting reports to txt: 47%|█████████████████████████████████████████▉ | 2383/5112 [55:53<03:52, 11.72it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3f8cee6d5bee5397.pdf\n", - "Converting reports to txt: 56%|██████████████████████████████████████████████████▋ | 2879/5112 [56:42<04:16, 8.72it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7e32023021d5aad2.pdf\n", - "Converting reports to txt: 59%|████████████████████████████████████████████████████▉ | 3010/5112 [56:58<03:20, 10.50it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b43100f226a61f53.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/b43100f226a61f53.pdf, using garbage: pdftoppm failed: 99\n", - "Converting reports to txt: 75%|███████████████████████████████████████████████████████████████████▌ | 3836/5112 [58:04<01:24, 15.13it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/6db03eff148934a4.pdf\n", - "Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/03cc150ce40cf816.pdf\n" + "Converting reports to txt: 41%|█████████████████████████████████████▏ | 2115/5115 [20:01<05:10, 9.65it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/69d373903e9635ef.pdf\n", + "Converting reports to txt: 42%|█████████████████████████████████████▋ | 2143/5115 [20:04<05:15, 9.43it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/ae1b80e9cccec491.pdf\n", + "Converting reports to txt: 43%|██████████████████████████████████████▎ | 2176/5115 [20:16<14:12, 3.45it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/21d5f1b05e1ba491.pdf\n", + "Converting reports to txt: 47%|██████████████████████████████████████████▏ | 2396/5115 [20:46<03:57, 11.46it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/3f8cee6d5bee5397.pdf\n", + "Converting reports to txt: 56%|██████████████████████████████████████████████████▋ | 2881/5115 [21:41<04:35, 8.10it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/7e32023021d5aad2.pdf\n", + "Converting reports to txt: 59%|█████████████████████████████████████████████████████ | 3017/5115 [22:16<03:37, 9.65it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/b43100f226a61f53.pdf\n", + "Converting reports to txt: 59%|█████████████████████████████████████████████████████▎ | 3031/5115 [22:17<03:21, 10.35it/s]Error during OCR of cc_04_10_2022/certs/reports/pdf/b43100f226a61f53.pdf, using garbage: pdftoppm failed: 99\n", + "Converting reports to txt: 75%|███████████████████████████████████████████████████████████████████▌ | 3840/5115 [23:32<01:42, 12.38it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/03cc150ce40cf816.pdf\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Error during OCR of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf, using garbage: pdftoppm failed: 1\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/6db03eff148934a4.pdf\n", + "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: f9c8da9deff77ab5 failed to convert report pdf->txt\n", - "Converting reports to txt: 76%|████████████████████████████████████████████████████████████████████▌ | 3896/5112 [58:08<01:13, 16.65it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3570791ff9c92912.pdf\n", - "Converting reports to txt: 77%|█████████████████████████████████████████████████████████████████████▍ | 3947/5112 [58:11<01:21, 14.22it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/ed886279d0d61096.pdf\n", - "Converting reports to txt: 86%|█████████████████████████████████████████████████████████████████████████████▍ | 4399/5112 [58:56<00:59, 11.91it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/c763b37bb95517e9.pdf\n", - "Converting reports to txt: 86%|█████████████████████████████████████████████████████████████████████████████▋ | 4416/5112 [58:57<00:54, 12.77it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/40640508ac3a37f2.pdf\n", - "Converting reports to txt: 87%|██████████████████████████████████████████████████████████████████████████████ | 4433/5112 [58:59<00:50, 13.40it/s]Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Converting reports to txt: 77%|████████████████████████████████████████████████████████████████████▊ | 3913/5115 [23:36<01:23, 14.40it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/3570791ff9c92912.pdf\n", + "Converting reports to txt: 77%|█████████████████████████████████████████████████████████████████████▌ | 3955/5115 [23:40<01:42, 11.27it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/ed886279d0d61096.pdf\n", + "Converting reports to txt: 86%|█████████████████████████████████████████████████████████████████████████████▌ | 4408/5115 [24:29<00:57, 12.26it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/c763b37bb95517e9.pdf\n", + "Converting reports to txt: 86%|█████████████████████████████████████████████████████████████████████████████▊ | 4422/5115 [24:30<00:55, 12.39it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/40640508ac3a37f2.pdf\n", + "Converting reports to txt: 87%|██████████████████████████████████████████████████████████████████████████████ | 4433/5115 [24:32<00:58, 11.59it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf\n", + "Converting reports to txt: 87%|██████████████████████████████████████████████████████████████████████████████▏ | 4447/5115 [24:33<00:54, 12.18it/s]Error during OCR of cc_04_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 7147c2f70d983d57 failed to convert report pdf->txt\n", - "Converting reports to txt: 95%|█████████████████████████████████████████████████████████████████████████████████████▎ | 4843/5112 [59:31<00:22, 12.22it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/3ad7749ef678ce95.pdf\n", - "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████ | 4888/5112 [59:34<00:14, 15.80it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/1683817e5db17d9c.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/b762fcb2983aa148.pdf\n", - "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████▍ | 4911/5112 [59:36<00:15, 12.93it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/cc681ca95621a6db.pdf\n", - "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████▋ | 4924/5112 [59:37<00:14, 12.83it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/889693d2b11bdfaf.pdf\n", - "Converting reports to txt: 97%|██████████████████████████████████████████████████████████████████████████████████████▉ | 4941/5112 [59:38<00:13, 12.37it/s]Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/876cb2fe02017f3f.pdf\n", - "Converting reports to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 5112/5112 [59:57<00:00, 1.42it/s]\n", - "Converting targets to txt: 2%|██ | 116/5034 [00:26<18:20, 4.47it/s]Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf\n", - "Error during OCR of cc_03_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: bfc86f14e2484335 failed to convert security target pdf->txt\n", - "Converting targets to txt: 10%|█████████▎ | 518/5034 [02:36<21:29, 3.50it/s]Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf\n", - "Converting targets to txt: 10%|█████████▍ | 525/5034 [02:41<32:08, 2.34it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf, using garbage: pdftoppm failed: 1\n", + "Converting reports to txt: 94%|████████████████████████████████████████████████████████████████████████████████████▉ | 4830/5115 [25:07<00:26, 10.62it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/3ad7749ef678ce95.pdf\n", + "Converting reports to txt: 96%|█████████████████████████████████████████████████████████████████████████████████████▉ | 4887/5115 [25:10<00:15, 15.18it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/b762fcb2983aa148.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/1683817e5db17d9c.pdf\n", + "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████▍ | 4915/5115 [25:12<00:16, 12.31it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/cc681ca95621a6db.pdf\n", + "Converting reports to txt: 96%|██████████████████████████████████████████████████████████████████████████████████████▋ | 4928/5115 [25:14<00:15, 11.81it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/889693d2b11bdfaf.pdf\n", + "Converting reports to txt: 97%|███████████████████████████████████████████████████████████████████████████████████████ | 4951/5115 [25:16<00:15, 10.92it/s]Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/876cb2fe02017f3f.pdf\n", + "Converting reports to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 5115/5115 [25:33<00:00, 3.34it/s]\n", + "Converting targets to txt: 2%|██ | 114/5032 [00:25<18:24, 4.45it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Converting targets to txt: 2%|██▏ | 121/5032 [00:27<18:58, 4.31it/s]Cert dgst: bfc86f14e2484335 failed to convert security target pdf->txt\n", + "Converting targets to txt: 10%|█████████▍ | 521/5032 [02:41<24:58, 3.01it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf\n", + "Converting targets to txt: 10%|█████████▌ | 528/5032 [02:47<34:23, 2.18it/s]Error during OCR of cc_04_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 4c9468f20fdb04f7 failed to convert security target pdf->txt\n", - "Converting targets to txt: 21%|███████████████████ | 1066/5034 [05:00<10:51, 6.09it/s]Error when converting pdf->txt: poppler error creating document\n", - "Converting targets to txt: 21%|███████████████████▏ | 1074/5034 [05:01<10:50, 6.08it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf\n", - "Converting targets to txt: 21%|███████████████████▎ | 1081/5034 [05:02<10:34, 6.23it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: a2b962c7f1d2bc56 failed to convert security target pdf->txt\n", - "Converting targets to txt: 28%|█████████████████████████▎ | 1414/5034 [06:30<13:27, 4.48it/s]Error when converting pdf->txt: poppler error creating document\n", - "Converting targets to txt: 28%|█████████████████████████▍ | 1422/5034 [06:33<17:06, 3.52it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf\n", - "Converting targets to txt: 28%|█████████████████████████▌ | 1429/5034 [06:35<15:29, 3.88it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Converting targets to txt: 21%|██████████████████▉ | 1059/5032 [05:08<11:45, 5.63it/s]Error when converting pdf->txt: poppler error creating document\n", + "Converting targets to txt: 21%|███████████████████ | 1067/5032 [05:09<12:12, 5.42it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf, using garbage: pdftoppm failed: 1\n", + "Converting targets to txt: 21%|███████████████████▏ | 1074/5032 [05:11<12:00, 5.50it/s]Cert dgst: a2b962c7f1d2bc56 failed to convert security target pdf->txt\n", + "Converting targets to txt: 28%|█████████████████████████▎ | 1417/5032 [06:43<15:03, 4.00it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf\n", + "Converting targets to txt: 28%|█████████████████████████▍ | 1424/5032 [06:46<17:46, 3.38it/s]Error during OCR of cc_04_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 422dc5758723c7d1 failed to convert security target pdf->txt\n", - "Converting targets to txt: 31%|████████████████████████████▎ | 1583/5034 [07:15<16:44, 3.44it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/063d3805a3b12930.pdf\n", - "Converting targets to txt: 33%|█████████████████████████████▌ | 1653/5034 [07:31<11:11, 5.03it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/2f4a41e8a5dfac6a.pdf\n", - "Converting targets to txt: 33%|██████████████████████████████ | 1684/5034 [07:36<10:06, 5.52it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/504258938266b87f.pdf\n", - "Converting targets to txt: 36%|███████████████████████████████▉ | 1789/5034 [08:01<11:47, 4.59it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/de85ff4c31069f12.pdf\n", - "Converting targets to txt: 37%|█████████████████████████████████ | 1851/5034 [08:16<12:16, 4.32it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/d3568613c552f9e8.pdf\n", - "Converting targets to txt: 37%|█████████████████████████████████▏ | 1855/5034 [08:17<12:45, 4.15it/s]Error during OCR of cc_03_10_2022/certs/targets/pdf/d3568613c552f9e8.pdf, using garbage: pdftoppm failed: 99\n", - "Converting targets to txt: 40%|████████████████████████████████████ | 2018/5034 [08:56<13:00, 3.86it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/b4bc79d8a558c7cf.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/09b43147f1307352.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/ce987167d42db722.pdf\n", - "Converting targets to txt: 40%|████████████████████████████████████▏ | 2022/5034 [08:58<13:34, 3.70it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/c16a92b40550193d.pdf\n", - "Converting targets to txt: 40%|███████████████████████████████████▌ | 2034/5034 [10:13<2:34:56, 3.10s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/2ff761edd4ed9b72.pdf\n", - "Converting targets to txt: 41%|███████████████████████████████████▉ | 2054/5034 [12:03<2:20:59, 2.84s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/9e4d3347efd95ec9.pdf\n", - "Converting targets to txt: 43%|███████████████████████████████████████ | 2185/5034 [12:42<13:25, 3.54it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/ad9ea5a4acf9ac53.pdf\n", - "Converting targets to txt: 44%|██████████████████████████████████████▎ | 2194/5034 [14:51<4:20:08, 5.50s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/28922ef97b2dfaa0.pdf\n", - "Converting targets to txt: 44%|█████████████████████████████████████▉ | 2196/5034 [17:07<17:33:53, 22.28s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/5c715225c805354f.pdf\n", - "Converting targets to txt: 68%|█████████████████████████████████████████████████████████████▌ | 3446/5034 [23:37<06:33, 4.04it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/731102b849e8d104.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/f65cab49ea5422dd.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/863d91c6a2b8e77a.pdf\n", - "Converting targets to txt: 77%|█████████████████████████████████████████████████████████████████████ | 3862/5034 [25:32<04:09, 4.69it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/18e5179ba124af1d.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/bc31fd0273ab72f8.pdf\n", - "Converting targets to txt: 79%|██████████████████████████████████████████████████████████████████████▉ | 3971/5034 [26:03<05:13, 3.39it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/5dfeb0219325989b.pdf\n", - "Converting targets to txt: 79%|███████████████████████████████████████████████████████████████████████ | 3975/5034 [26:05<05:21, 3.29it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/22617e916bad6f53.pdf\n", - "Converting targets to txt: 79%|███████████████████████████████████████████████████████████████████████ | 3978/5034 [26:06<05:33, 3.17it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/6344672e3178f63e.pdf\n", - "Converting targets to txt: 80%|███████████████████████████████████████████████████████████████████████▊ | 4014/5034 [32:17<44:48, 2.64s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/483dc36fcf11b257.pdf\n", - "Converting targets to txt: 80%|██████████████████████████████████████████████████████████████████████▎ | 4024/5034 [33:13<1:15:50, 4.51s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/5885f28f042038c1.pdf\n", - "Converting targets to txt: 80%|████████████████████████████████████████████████████████████████████████▎ | 4048/5034 [33:36<16:35, 1.01s/it]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/7b08ad74a882337c.pdf\n", - "Converting targets to txt: 87%|██████████████████████████████████████████████████████████████████████████████▎ | 4380/5034 [36:12<02:11, 4.97it/s]Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf\n", - "Error during OCR of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Converting targets to txt: 31%|████████████████████████████ | 1572/5032 [07:26<18:40, 3.09it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/063d3805a3b12930.pdf\n", + "Converting targets to txt: 33%|█████████████████████████████▌ | 1651/5032 [07:45<12:10, 4.63it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/2f4a41e8a5dfac6a.pdf\n", + "Converting targets to txt: 33%|█████████████████████████████▉ | 1676/5032 [07:50<11:36, 4.82it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/504258938266b87f.pdf\n", + "Converting targets to txt: 35%|███████████████████████████████▊ | 1781/5032 [08:18<13:29, 4.02it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/de85ff4c31069f12.pdf\n", + "Converting targets to txt: 37%|█████████████████████████████████ | 1850/5032 [08:37<13:22, 3.96it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/d3568613c552f9e8.pdf\n", + "Converting targets to txt: 37%|█████████████████████████████████▏ | 1858/5032 [08:42<22:28, 2.35it/s]Error during OCR of cc_04_10_2022/certs/targets/pdf/d3568613c552f9e8.pdf, using garbage: pdftoppm failed: 99\n", + "Converting targets to txt: 40%|████████████████████████████████████ | 2014/5032 [09:20<13:10, 3.82it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/b4bc79d8a558c7cf.pdf\n", + "Converting targets to txt: 40%|████████████████████████████████████ | 2017/5032 [09:22<16:54, 2.97it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/09b43147f1307352.pdf\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/ce987167d42db722.pdf\n", + "Converting targets to txt: 40%|████████████████████████████████████▏ | 2020/5032 [09:23<17:00, 2.95it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/c16a92b40550193d.pdf\n", + "Converting targets to txt: 40%|███████████████████████████████████▌ | 2032/5032 [10:42<2:44:41, 3.29s/it]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/2ff761edd4ed9b72.pdf\n", + "Converting targets to txt: 41%|███████████████████████████████████▊ | 2050/5032 [12:33<2:34:26, 3.11s/it]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/9e4d3347efd95ec9.pdf\n", + "Converting targets to txt: 43%|███████████████████████████████████████ | 2182/5032 [13:38<14:26, 3.29it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/ad9ea5a4acf9ac53.pdf\n", + "Converting targets to txt: 44%|██████████████████████████████████████▎ | 2192/5032 [16:39<6:57:28, 8.82s/it]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/28922ef97b2dfaa0.pdf\n", + "Converting targets to txt: 44%|█████████████████████████████████████▉ | 2194/5032 [20:07<29:34:31, 37.52s/it]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/5c715225c805354f.pdf\n", + "Converting targets to txt: 68%|█████████████████████████████████████████████████████████████▍ | 3437/5032 [27:19<06:05, 4.36it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/863d91c6a2b8e77a.pdf\n", + "Converting targets to txt: 68%|█████████████████████████████████████████████████████████████▌ | 3445/5032 [27:21<06:55, 3.82it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/f65cab49ea5422dd.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/731102b849e8d104.pdf\n", + "Converting targets to txt: 77%|████████████████████████████████████████████████████████████████████▉ | 3856/5032 [29:22<04:34, 4.28it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/bc31fd0273ab72f8.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/18e5179ba124af1d.pdf\n", + "Converting targets to txt: 79%|██████████████████████████████████████████████████████████████████████▉ | 3968/5032 [29:56<05:36, 3.16it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/5dfeb0219325989b.pdf\n", + "Converting targets to txt: 79%|███████████████████████████████████████████████████████████████████████ | 3974/5032 [29:59<06:06, 2.89it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/22617e916bad6f53.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/6344672e3178f63e.pdf\n", + "Converting targets to txt: 80%|███████████████████████████████████████████████████████████████████████▊ | 4016/5032 [38:07<29:43, 1.76s/it]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/483dc36fcf11b257.pdf\n", + "Converting targets to txt: 80%|██████████████████████████████████████████████████████████████████████▎ | 4022/5032 [39:24<1:31:32, 5.44s/it]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/5885f28f042038c1.pdf\n", + "Converting targets to txt: 80%|████████████████████████████████████████████████████████████████████████▎ | 4046/5032 [39:36<14:33, 1.13it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/7b08ad74a882337c.pdf\n", + "Converting targets to txt: 87%|██████████████████████████████████████████████████████████████████████████████▎ | 4378/5032 [42:42<02:31, 4.31it/s]Error when converting pdf->txt: poppler error creating document\n", + "Converting targets to txt: 87%|██████████████████████████████████████████████████████████████████████████████▍ | 4383/5032 [42:44<02:33, 4.22it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 7147c2f70d983d57 failed to convert security target pdf->txt\n", - "Converting targets to txt: 90%|████████████████████████████████████████████████████████████████████████████████▉ | 4525/5034 [36:56<02:32, 3.33it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/931e1b2a68a1bcd2.pdf\n", - "Converting targets to txt: 92%|██████████████████████████████████████████████████████████████████████████████████▌ | 4615/5034 [37:15<01:37, 4.31it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/16229c97ce083e8c.pdf\n", - "Converting targets to txt: 99%|████████████████████████████████████████████████████████████████████████████████████████▉ | 4977/5034 [38:46<00:13, 4.22it/s]Error when converting pdf->txt: poppler error creating page\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/a552244ec3ca8f60.pdf\n", - "Converting targets to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 5034/5034 [40:44<00:00, 2.06it/s]\n", + "Converting targets to txt: 90%|████████████████████████████████████████████████████████████████████████████████▉ | 4523/5032 [43:33<03:02, 2.79it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/931e1b2a68a1bcd2.pdf\n", + "Converting targets to txt: 92%|██████████████████████████████████████████████████████████████████████████████████▍ | 4612/5032 [43:54<01:46, 3.93it/s]Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/16229c97ce083e8c.pdf\n", + "Converting targets to txt: 99%|████████████████████████████████████████████████████████████████████████████████████████▉ | 4973/5032 [45:28<00:13, 4.30it/s]Error when converting pdf->txt: poppler error creating page\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/a552244ec3ca8f60.pdf\n", + "Converting targets to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 5032/5032 [47:33<00:00, 1.76it/s]\n", "Converting reports to txt: 0%| | 0/8 [00:00<?, ?it/s]Error when converting pdf->txt: poppler error creating document\n", "Error when converting pdf->txt: poppler error creating document\n", "Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf\n", "Error when converting pdf->txt: poppler error creating document\n", "Error when converting pdf->txt: poppler error creating document\n", "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf\n", "Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf\n", "Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/02958808b57334f0.pdf, using garbage: pdftoppm failed: 1\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/4de4a9f436958574.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/02958808b57334f0.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 7147c2f70d983d57 failed to convert report pdf->txt\n", + "Cert dgst: 2d05e976e1adf696 failed to convert report pdf->txt\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 98668cafb8dda26f failed to convert report pdf->txt\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/02958808b57334f0.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 02958808b57334f0 failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 422dc5758723c7d1 failed to convert report pdf->txt\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: bfc86f14e2484335 failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/f9c8da9deff77ab5.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: f9c8da9deff77ab5 failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: 422dc5758723c7d1 failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: 7147c2f70d983d57 failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/98668cafb8dda26f.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: 98668cafb8dda26f failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/2d05e976e1adf696.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: 2d05e976e1adf696 failed to convert report pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/reports/pdf/4de4a9f436958574.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: 4de4a9f436958574 failed to convert report pdf->txt\n", - "Converting reports to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:01<00:00, 7.98it/s]\n", - "Error when converting pdf->txt: poppler error creating document\n", - "Converting targets to txt: 0%| | 0/5 [00:00<?, ?it/s]Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf\n" + "Error during OCR of cc_04_10_2022/certs/reports/pdf/4de4a9f436958574.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: 4de4a9f436958574 failed to convert report pdf->txt\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ + "Converting reports to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:01<00:00, 7.98it/s]\n", + "Converting targets to txt: 0%| | 0/5 [00:00<?, ?it/s]Error when converting pdf->txt: poppler error creating document\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf\n", "Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf\n", "Error when converting pdf->txt: poppler error creating document\n", "Error when converting pdf->txt: poppler error creating document\n", "Error when converting pdf->txt: poppler error creating document\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf\n", - "Detected garbage during conversion of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf\n", - "Error during OCR of cc_03_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf, using garbage: pdftoppm failed: 1\n", - "Cert dgst: a2b962c7f1d2bc56 failed to convert security target pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/7147c2f70d983d57.pdf, using garbage: pdftoppm failed: 1\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf\n", + "Detected garbage during conversion of cc_04_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf\n", "Cert dgst: 7147c2f70d983d57 failed to convert security target pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/a2b962c7f1d2bc56.pdf, using garbage: pdftoppm failed: 1\n", + "Cert dgst: a2b962c7f1d2bc56 failed to convert security target pdf->txt\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/bfc86f14e2484335.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: bfc86f14e2484335 failed to convert security target pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/422dc5758723c7d1.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 422dc5758723c7d1 failed to convert security target pdf->txt\n", - "Error during OCR of cc_03_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf, using garbage: pdftoppm failed: 1\n", + "Error during OCR of cc_04_10_2022/certs/targets/pdf/4c9468f20fdb04f7.pdf, using garbage: pdftoppm failed: 1\n", "Cert dgst: 4c9468f20fdb04f7 failed to convert security target pdf->txt\n", "Converting targets to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:01<00:00, 4.99it/s]\n" ] @@ -471,7 +483,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 12, "metadata": { "pycharm": { "name": "#%%\n" @@ -483,10 +495,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "Extracting report metadata: 0%| | 0/5104 [00:00<?, ?it/s]Failed to read metadata of certs/reports/pdf/1e91595cd5e7fece.pdf, error: Unexpected escaped string: b'\\x83'\n", + "Extracting report metadata: 0%| | 0/5107 [00:00<?, ?it/s]Failed to read metadata of certs/reports/pdf/1e91595cd5e7fece.pdf, error: Unexpected escaped string: b'\\x83'\n", "Failed to read metadata of certs/reports/pdf/451e0806dfcfd29d.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 11%|██████████ | 574/5104 [00:02<00:15, 288.04it/s]Failed to read metadata of certs/reports/pdf/e51bbdd7e3705d61.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 17%|███████████████▍ | 879/5104 [00:06<00:48, 86.25it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x59564 for key /Im21 [generic.py:590]\n", + "Extracting report metadata: 14%|████████████▌ | 720/5107 [00:03<00:20, 217.24it/s]Failed to read metadata of certs/reports/pdf/e51bbdd7e3705d61.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 17%|███████████████▋ | 889/5107 [00:09<01:48, 38.76it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x59564 for key /Im21 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x59571 for key /Im21 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x5957e for key /Im21 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x5a3f9 for key /Im21 [generic.py:590]\n", @@ -531,65 +543,65 @@ "PdfReadWarning: Multiple definitions in dictionary at byte 0x82211 for key /Im21 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x8221e for key /Im21 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x8222b for key /Im21 [generic.py:590]\n", - "Extracting report metadata: 26%|███████████████████████ | 1339/5104 [00:09<00:25, 147.71it/s]Failed to read metadata of certs/reports/pdf/dc54e899ab191a2a.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 34%|██████████████████████████████ | 1742/5104 [00:11<00:20, 165.40it/s]Failed to read metadata of certs/reports/pdf/a62c55533f4e1636.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 28%|████████████████████████▋ | 1430/5107 [00:13<00:28, 128.36it/s]Failed to read metadata of certs/reports/pdf/dc54e899ab191a2a.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 35%|██████████████████████████████▊ | 1786/5107 [00:15<00:22, 146.12it/s]Failed to read metadata of certs/reports/pdf/a62c55533f4e1636.pdf, error: PDF metadata unavailable\n", "Failed to read metadata of certs/reports/pdf/d3568613c552f9e8.pdf, error: EOF marker not found\n", - "Extracting report metadata: 37%|████████████████████████████████▉ | 1912/5104 [00:12<00:19, 165.35it/s]Failed to read metadata of certs/reports/pdf/ea5ffdcf17f6ecf4.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 45%|███████████████████████████████████████▍ | 2290/5104 [00:14<00:16, 173.82it/s]Failed to read metadata of certs/reports/pdf/1c5deb472a197eb6.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 38%|█████████████████████████████████▋ | 1956/5107 [00:16<00:20, 151.10it/s]Failed to read metadata of certs/reports/pdf/ea5ffdcf17f6ecf4.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 45%|████████████████████████████████████████ | 2323/5107 [00:18<00:16, 165.54it/s]Failed to read metadata of certs/reports/pdf/1c5deb472a197eb6.pdf, error: PDF metadata unavailable\n", "Failed to read metadata of certs/reports/pdf/0f3295c420339dd2.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 50%|███████████████████████████████████████████▉ | 2551/5104 [00:15<00:12, 197.33it/s]Failed to read metadata of certs/reports/pdf/6be07dff9c4ed819.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 56%|█████████████████████████████████████████████████ | 2849/5104 [00:16<00:10, 224.61it/s]Failed to read metadata of certs/reports/pdf/b43100f226a61f53.pdf, error: EOF marker not found\n", - "Extracting report metadata: 61%|█████████████████████████████████████████████████████▋ | 3115/5104 [00:17<00:08, 233.73it/s]Failed to read metadata of certs/reports/pdf/be0806db6fd6bbe8.pdf, error: 'NumberObject' object is not subscriptable\n", + "Extracting report metadata: 50%|████████████████████████████████████████████ | 2560/5107 [00:19<00:13, 184.46it/s]Failed to read metadata of certs/reports/pdf/6be07dff9c4ed819.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 55%|████████████████████████████████████████████████▍ | 2814/5107 [00:20<00:11, 202.18it/s]Failed to read metadata of certs/reports/pdf/b43100f226a61f53.pdf, error: EOF marker not found\n", + "Extracting report metadata: 63%|███████████████████████████████████████████████████████▌ | 3225/5107 [00:22<00:09, 196.09it/s]Failed to read metadata of certs/reports/pdf/be0806db6fd6bbe8.pdf, error: 'NumberObject' object is not subscriptable\n", "Failed to read metadata of certs/reports/pdf/35ac4cfd1f62df75.pdf, error: 'NumberObject' object is not subscriptable\n", "Failed to read metadata of certs/reports/pdf/aaf48074ea1b4bf4.pdf, error: invalid literal for int() with base 10: b'n'\n", "Failed to read metadata of certs/reports/pdf/206662b82595265b.pdf, error: 'NumberObject' object is not subscriptable\n", "Failed to read metadata of certs/reports/pdf/989b106f5e63b4d1.pdf, error: invalid literal for int() with base 10: b'n'\n", "Failed to read metadata of certs/reports/pdf/0160f6becf137107.pdf, error: 'NumberObject' object is not subscriptable\n", - "Extracting report metadata: 65%|████████████████████████████████████████████████████████▊ | 3294/5104 [00:18<00:08, 215.65it/s]Failed to read metadata of certs/reports/pdf/9558a9eaadae855e.pdf, error: PDF metadata unavailable\n" + "Failed to read metadata of certs/reports/pdf/9558a9eaadae855e.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 74%|████████████████████████████████████████████████████████████████▉ | 3772/5107 [00:29<00:12, 107.74it/s]Failed to read metadata of certs/reports/pdf/9114f0f9938e558c.pdf, error: PDF metadata unavailable\n", + "Failed to read metadata of certs/reports/pdf/08b09322ee9df0af.pdf, error: PDF metadata unavailable\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Extracting report metadata: 72%|███████████████████████████████████████████████████████████████ | 3657/5104 [00:22<00:12, 118.13it/s]Failed to read metadata of certs/reports/pdf/9114f0f9938e558c.pdf, error: PDF metadata unavailable\n", - "Failed to read metadata of certs/reports/pdf/08b09322ee9df0af.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 77%|███████████████████████████████████████████████████████████████████▊ | 3935/5104 [00:23<00:07, 163.39it/s]Failed to read metadata of certs/reports/pdf/518b3066ebfabcd3.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 85%|██████████████████████████████████████████████████████████████████████████▊ | 4339/5104 [00:25<00:04, 180.47it/s]Failed to read metadata of certs/reports/pdf/d455e5408b744b44.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 100%|████████████████████████████████████████████████████████████████████████████████████████| 5104/5104 [00:32<00:00, 158.98it/s]\n", - "Extracting target metadata: 2%|█▍ | 78/5029 [00:01<01:06, 74.44it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x724b1 for key /Im312 [generic.py:590]\n", + "Failed to read metadata of certs/reports/pdf/518b3066ebfabcd3.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 87%|████████████████████████████████████████████████████████████████████████████▊ | 4459/5107 [00:32<00:03, 181.19it/s]Failed to read metadata of certs/reports/pdf/d455e5408b744b44.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 100%|████████████████████████████████████████████████████████████████████████████████████████| 5107/5107 [00:38<00:00, 133.02it/s]\n", + "Extracting target metadata: 1%|▊ | 45/5027 [00:01<01:58, 42.14it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x724b1 for key /Im312 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x724cf for key /Im312 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x724de for key /Im312 [generic.py:590]\n", "Failed to read metadata of certs/targets/pdf/d47a03be1cb342b1.pdf, error: PDF metadata unavailable\n", - "Failed to read metadata of certs/targets/pdf/5af5a1b535422b51.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 2%|██▏ | 120/5027 [00:02<01:23, 58.92it/s]Failed to read metadata of certs/targets/pdf/5af5a1b535422b51.pdf, error: PDF metadata unavailable\n", "Failed to read metadata of certs/targets/pdf/21d6eff0bfe3971f.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 4%|███▉ | 219/5029 [00:04<01:47, 44.54it/s]Failed to read metadata of certs/targets/pdf/eee469bd6d07f8db.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 6%|█████▋ | 317/5029 [00:08<02:52, 27.27it/s]Failed to read metadata of certs/targets/pdf/c62b878b1ac43df9.pdf, error: PDF metadata unavailable\n", - "Failed to read metadata of certs/targets/pdf/4c017087976d05eb.pdf, error: EOF marker not found\n", - "Extracting target metadata: 7%|██████ | 339/5029 [00:09<03:04, 25.42it/s]Failed to read metadata of certs/targets/pdf/0dc53ac916660b90.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 4%|███▊ | 212/5027 [00:05<02:24, 33.42it/s]Failed to read metadata of certs/targets/pdf/eee469bd6d07f8db.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 6%|█████▋ | 320/5027 [00:10<03:30, 22.32it/s]Failed to read metadata of certs/targets/pdf/c62b878b1ac43df9.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 7%|█████▉ | 331/5027 [00:11<04:10, 18.73it/s]Failed to read metadata of certs/targets/pdf/4c017087976d05eb.pdf, error: EOF marker not found\n", "Failed to read metadata of certs/targets/pdf/e676143c80802a59.pdf, error: EOF marker not found\n", "Failed to read metadata of certs/targets/pdf/e6e8add5e4db2d9d.pdf, error: EOF marker not found\n", "Failed to read metadata of certs/targets/pdf/1c9bd4db3d8388f7.pdf, error: EOF marker not found\n", - "Extracting target metadata: 7%|██████▌ | 364/5029 [00:10<03:06, 25.06it/s]Failed to read metadata of certs/targets/pdf/4bede5d2af4cb11f.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 9%|████████ | 452/5029 [00:13<02:44, 27.77it/s]Failed to read metadata of certs/targets/pdf/d2f75cdaeb322e7f.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 10%|████████▌ | 478/5029 [00:14<02:48, 26.97it/s]Failed to read metadata of certs/targets/pdf/c9b0cae8b5784d2d.pdf, error: PDF metadata unavailable\n", - "Failed to read metadata of certs/targets/pdf/af6767c887ae930a.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 10%|█████████▏ | 515/5029 [00:15<02:32, 29.60it/s]Failed to read metadata of certs/targets/pdf/42d2e68c29d9eef7.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 12%|██████████▊ | 605/5029 [00:17<02:00, 36.65it/s]Failed to read metadata of certs/targets/pdf/44f1f74c09e634cb.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 13%|███████████▌ | 648/5029 [00:18<01:54, 38.12it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x2ee44 for key /Im59 [generic.py:590]\n", + "Failed to read metadata of certs/targets/pdf/4bede5d2af4cb11f.pdf, error: PDF metadata unavailable\n", + "Failed to read metadata of certs/targets/pdf/0dc53ac916660b90.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 9%|████████▎ | 461/5027 [00:16<03:27, 21.96it/s]Failed to read metadata of certs/targets/pdf/af6767c887ae930a.pdf, error: PDF metadata unavailable\n", + "Failed to read metadata of certs/targets/pdf/d2f75cdaeb322e7f.pdf, error: PDF metadata unavailable\n", + "Failed to read metadata of certs/targets/pdf/c9b0cae8b5784d2d.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 10%|█████████▏ | 513/5027 [00:18<03:15, 23.09it/s]Failed to read metadata of certs/targets/pdf/42d2e68c29d9eef7.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 12%|██████████▌ | 593/5027 [00:20<02:23, 30.81it/s]Failed to read metadata of certs/targets/pdf/44f1f74c09e634cb.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 13%|███████████▎ | 633/5027 [00:21<02:12, 33.13it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x2ee44 for key /Im59 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x6f7cf for key /Im215 [generic.py:590]\n", - "Extracting target metadata: 19%|█████████████████▍ | 975/5029 [00:32<02:32, 26.58it/s]Failed to read metadata of certs/targets/pdf/3183166f4cb1c37c.pdf, error: PDF metadata unavailable\n", - "Failed to read metadata of certs/targets/pdf/b224a05ad0f1ec6b.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 19%|████████████████▉ | 946/5027 [00:37<03:25, 19.90it/s]Failed to read metadata of certs/targets/pdf/3183166f4cb1c37c.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 20%|█████████████████▊ | 995/5027 [00:38<02:23, 28.08it/s]Failed to read metadata of certs/targets/pdf/b224a05ad0f1ec6b.pdf, error: PDF metadata unavailable\n", "Failed to read metadata of certs/targets/pdf/0cb80431100b4bd4.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 21%|██████████████████▊ | 1060/5029 [00:33<01:32, 43.08it/s]Failed to read metadata of certs/targets/pdf/14ddeb255e0a1cd2.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 23%|████████████████████▍ | 1157/5029 [00:34<01:06, 58.27it/s]Failed to read metadata of certs/targets/pdf/0c5a0893bb4f17dc.pdf, error: PDF metadata unavailable\n", - "Failed to read metadata of certs/targets/pdf/c13a0ff38bf1cf86.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 25%|█████████████████████▉ | 1237/5029 [00:35<00:59, 64.22it/s]Failed to read metadata of certs/targets/pdf/e84d854109fc9e59.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 21%|██████████████████▊ | 1064/5027 [00:39<01:40, 39.60it/s]Failed to read metadata of certs/targets/pdf/14ddeb255e0a1cd2.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 23%|████████████████████▍ | 1155/5027 [00:40<01:11, 54.04it/s]Failed to read metadata of certs/targets/pdf/0c5a0893bb4f17dc.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 24%|█████████████████████▋ | 1227/5027 [00:41<01:04, 58.58it/s]Failed to read metadata of certs/targets/pdf/c13a0ff38bf1cf86.pdf, error: PDF metadata unavailable\n", + "Failed to read metadata of certs/targets/pdf/e84d854109fc9e59.pdf, error: PDF metadata unavailable\n", "Failed to read metadata of certs/targets/pdf/e39bcd3ef3ba2f00.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 26%|███████████████████████▍ | 1321/5029 [00:37<01:12, 51.45it/s]Failed to read metadata of certs/targets/pdf/af478678c6464da0.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 31%|███████████████████████████▉ | 1581/5029 [00:44<01:24, 40.86it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x21033 for key /Im67 [generic.py:590]\n", + "Extracting target metadata: 26%|███████████████████████▏ | 1308/5027 [00:43<01:16, 48.60it/s]Failed to read metadata of certs/targets/pdf/af478678c6464da0.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 31%|███████████████████████████▌ | 1556/5027 [00:50<01:27, 39.64it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x21033 for key /Im67 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x21067 for key /Im67 [generic.py:590]\n", - "PdfReadWarning: Multiple definitions in dictionary at byte 0x21074 for key /Im68 [generic.py:590]\n", + "Extracting target metadata: 31%|███████████████████████████▉ | 1578/5027 [00:51<01:40, 34.18it/s]PdfReadWarning: Multiple definitions in dictionary at byte 0x21074 for key /Im68 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x21081 for key /Im69 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x2108e for key /Im67 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x210b5 for key /Im67 [generic.py:590]\n", @@ -604,16 +616,16 @@ "PdfReadWarning: Multiple definitions in dictionary at byte 0x3ec89 for key /Im135 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x3eca7 for key /Im145 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x3ed10 for key /Im143 [generic.py:590]\n", - "PdfReadWarning: Multiple definitions in dictionary at byte 0x3ed1f for key /Im149 [generic.py:590]\n" + "PdfReadWarning: Multiple definitions in dictionary at byte 0x3ed1f for key /Im149 [generic.py:590]\n", + "PdfReadWarning: Multiple definitions in dictionary at byte 0x61908 for key /Im173 [generic.py:590]\n", + "PdfReadWarning: Multiple definitions in dictionary at byte 0x619e9 for key /Im200 [generic.py:590]\n", + "PdfReadWarning: Multiple definitions in dictionary at byte 0x61a16 for key /Im205 [generic.py:590]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "PdfReadWarning: Multiple definitions in dictionary at byte 0x61908 for key /Im173 [generic.py:590]\n", - "PdfReadWarning: Multiple definitions in dictionary at byte 0x619e9 for key /Im200 [generic.py:590]\n", - "PdfReadWarning: Multiple definitions in dictionary at byte 0x61a16 for key /Im205 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x61b7e for key /Im220 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x9146a for key /Im324 [generic.py:590]\n", "PdfReadWarning: Multiple definitions in dictionary at byte 0x91479 for key /Im324 [generic.py:590]\n", @@ -632,54 +644,57 @@ "Failed to read metadata of certs/targets/pdf/bbcfe0aee7ea78ea.pdf, error: PDF metadata unavailable\n", "Failed to read metadata of certs/targets/pdf/f0bd6a29eee94a2c.pdf, error: PDF metadata unavailable\n", "Failed to read metadata of certs/targets/pdf/463ecd64b7506048.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 36%|████████████████████████████████ | 1815/5029 [00:47<00:49, 64.98it/s]Failed to read metadata of certs/targets/pdf/d3568613c552f9e8.pdf, error: EOF marker not found\n", - "Extracting target metadata: 42%|████████████████████████████████████▉ | 2089/5029 [00:50<00:36, 79.70it/s]UserWarning: startxref on same line as offset [pdf.py:1787]\n", - "Extracting target metadata: 44%|███████████████████████████████████████▎ | 2223/5029 [00:52<00:38, 73.68it/s]Failed to read metadata of certs/targets/pdf/4d2a177384b23fd6.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 59%|████████████████████████████████████████████████████ | 2945/5029 [01:12<00:46, 44.62it/s]Failed to read metadata of certs/targets/pdf/a83fe0ea2f391ee8.pdf, error: too many values to unpack (expected 2)\n", - "Extracting target metadata: 63%|████████████████████████████████████████████████████████▏ | 3178/5029 [01:17<00:43, 42.20it/s]Failed to read metadata of certs/targets/pdf/907433fbf9e00ce3.pdf, error: Could not find xref table at specified location\n", - "Extracting target metadata: 96%|█████████████████████████████████████████████████████████████████████████████████████▌ | 4837/5029 [01:40<00:02, 95.37it/s]Failed to read metadata of certs/targets/pdf/fc8d2eade14aa446.pdf, error: 'NumberObject' object is not subscriptable\n", - "Extracting target metadata: 98%|███████████████████████████████████████████████████████████████████████████████████████▌ | 4946/5029 [01:42<00:01, 72.77it/s]Failed to read metadata of certs/targets/pdf/a334142059d865a2.pdf, error: PDF metadata unavailable\n", - "Extracting target metadata: 100%|████████████████████████████████████████████████████████████████████████████████████████▊| 5021/5029 [01:43<00:00, 72.76it/s]Failed to read metadata of certs/targets/pdf/a552244ec3ca8f60.pdf, error: EOF marker not found\n", - "Extracting target metadata: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 5029/5029 [01:45<00:00, 47.68it/s]\n", - "Extracting report frontpages: 8%|██████▉ | 404/5104 [00:01<00:12, 384.61it/s]WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/4844355021ea22f1.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/4844355021ea22f1.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/d8aff63d56506605.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/21a1deda31c3e213.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/aca070da96add3b6.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/d8aff63d56506605.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/aca070da96add3b6.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/21a1deda31c3e213.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/977b04d04def60a7.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/977b04d04def60a7.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/6eeed6fefb1f0243.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/6eeed6fefb1f0243.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/62313881916550ca.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/6f8d7a6a1dea6a3a.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/62313881916550ca.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/6f8d7a6a1dea6a3a.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/b0b7b073ca2dfe5f.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/47913a485c3c8a18.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/60f3bd10ee9be85b.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/47913a485c3c8a18.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/b0b7b073ca2dfe5f.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/60f3bd10ee9be85b.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/5c3806bf79eeab7f.txt\n", - "WARNING: multiple rules are matching same certification document: /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/reports/txt/5c3806bf79eeab7f.txt\n" + "Extracting target metadata: 35%|███████████████████████████████▌ | 1784/5027 [00:55<00:59, 54.46it/s]Failed to read metadata of certs/targets/pdf/d3568613c552f9e8.pdf, error: EOF marker not found\n", + "Extracting target metadata: 42%|█████████████████████████████████████▍ | 2114/5027 [00:59<00:40, 72.39it/s]UserWarning: startxref on same line as offset [pdf.py:1787]\n", + "Extracting target metadata: 45%|████████████████████████████████████████ | 2263/5027 [01:02<00:48, 56.91it/s]Failed to read metadata of certs/targets/pdf/4d2a177384b23fd6.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 59%|████████████████████████████████████████████████████▏ | 2950/5027 [01:25<00:56, 36.82it/s]Failed to read metadata of certs/targets/pdf/a83fe0ea2f391ee8.pdf, error: too many values to unpack (expected 2)\n", + "Extracting target metadata: 63%|███████████████████████████████████████████████████████▉ | 3158/5027 [01:30<00:49, 37.54it/s]Failed to read metadata of certs/targets/pdf/907433fbf9e00ce3.pdf, error: Could not find xref table at specified location\n", + "Extracting target metadata: 83%|██████████████████████████████████████████████████████████████████████████▏ | 4193/5027 [01:48<00:14, 59.32it/s]UserWarning: startxref on same line as offset [pdf.py:1787]\n", + "Extracting target metadata: 97%|██████████████████████████████████████████████████████████████████████████████████████▏ | 4871/5027 [01:58<00:02, 73.51it/s]Failed to read metadata of certs/targets/pdf/fc8d2eade14aa446.pdf, error: 'NumberObject' object is not subscriptable\n", + "Extracting target metadata: 98%|███████████████████████████████████████████████████████████████████████████████████████▌ | 4947/5027 [02:00<00:01, 54.78it/s]Failed to read metadata of certs/targets/pdf/a334142059d865a2.pdf, error: PDF metadata unavailable\n", + "Extracting target metadata: 100%|████████████████████████████████████████████████████████████████████████████████████████▉| 5026/5027 [02:02<00:00, 44.40it/s]Failed to read metadata of certs/targets/pdf/a552244ec3ca8f60.pdf, error: EOF marker not found\n", + "Extracting target metadata: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 5027/5027 [02:03<00:00, 40.74it/s]\n", + "Extracting report frontpages: 8%|███████ | 412/5107 [00:01<00:12, 390.40it/s]WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/21a1deda31c3e213.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/d8aff63d56506605.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/aca070da96add3b6.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/4844355021ea22f1.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/21a1deda31c3e213.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/d8aff63d56506605.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/aca070da96add3b6.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/4844355021ea22f1.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/977b04d04def60a7.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/6eeed6fefb1f0243.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/977b04d04def60a7.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/6eeed6fefb1f0243.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/6f8d7a6a1dea6a3a.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/6f8d7a6a1dea6a3a.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/62313881916550ca.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/62313881916550ca.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/47913a485c3c8a18.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/47913a485c3c8a18.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/60f3bd10ee9be85b.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/b0b7b073ca2dfe5f.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/5c3806bf79eeab7f.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/b0b7b073ca2dfe5f.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/60f3bd10ee9be85b.txt\n", + "WARNING: multiple rules are matching same certification document: cc_04_10_2022/certs/reports/txt/5c3806bf79eeab7f.txt\n", + "Extracting report frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████| 5107/5107 [00:14<00:00, 351.07it/s]\n", + "Extracting target frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████| 5027/5027 [00:22<00:00, 228.40it/s]\n", + "Extracting report keywords: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 5107/5107 [10:53<00:00, 7.81it/s]\n", + "Extracting target keywords: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 5027/5027 [39:51<00:00, 2.10it/s]\n", + "Fitting the CPE classifier: 100%|█████████████████████████████████████████████████████████████████████████████████| 888900/888900 [00:01<00:00, 855040.09it/s]\n", + "Predicting CPE matches with the classifier: 100%|█████████████████████████████████████████████████████████████████████████| 5129/5129 [03:20<00:00, 25.64it/s]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Extracting report frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████| 5104/5104 [00:13<00:00, 374.77it/s]\n", - "Extracting target frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████| 5029/5029 [00:18<00:00, 269.71it/s]\n", - "Extracting report keywords: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 5104/5104 [08:49<00:00, 9.64it/s]\n", - "Extracting target keywords: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5029/5029 [32:59<00:00, 2.54it/s]\n", - "Fitting the CPE classifier: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 888243/888243 [00:02<00:00, 413368.35it/s]\n", - "Predicting CPE matches with the classifier: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5126/5126 [02:53<00:00, 29.56it/s]\n", - "parsing cpe matching (by NIST) dictionary: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 397693/397693 [00:26<00:00, 15145.61it/s]\n", - "Building-up lookup dictionaries for fast CVE matching: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████| 196860/196860 [00:09<00:00, 19964.38it/s]\n", - "Computing related CVES: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 704/704 [00:00<00:00, 79643.70it/s]\n", + "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 21/21 [00:11<00:00, 1.81it/s]\n", + "Building CVEDataset from jsons: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 21/21 [01:25<00:00, 4.09s/it]\n", + "parsing cpe matching (by NIST) dictionary: 100%|███████████████████████████████████████████████████████████████████| 398394/398394 [00:27<00:00, 14339.71it/s]\n", + "Building-up lookup dictionaries for fast CVE matching: 100%|███████████████████████████████████████████████████████| 196907/196907 [00:12<00:00, 16078.76it/s]\n", + "Computing related CVES: 100%|████████████████████████████████████████████████████████████████████████████████████████████| 698/698 [00:00<00:00, 82482.23it/s]\n", "WARNING:root:There were total of 153 certificates skipped due to duplicity\n" ] } @@ -691,7 +706,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 13, "metadata": { "pycharm": { "name": "#%%\n" @@ -702,61 +717,44 @@ "name": "stderr", "output_type": "stream", "text": [ - "Downloading reports: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 457/457 [01:40<00:00, 4.56it/s]\n", - "Downloading targets: 41%|██████████████████████████████████████████████████████████████▌ | 188/457 [01:13<03:08, 1.43it/s]ERROR:sec_certs.sample.certificate:Cert dgst: cert_4d38305c4a2ee035_update_d37e22d00eafccc0 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/0976ra1b_pdf.pdf, code: 408\n", - "ERROR:sec_certs.sample.certificate:Cert dgst: cert_bcac7e3f9f614ec6_update_531b2055de9b34f2 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/XSMART%20e-Passport%20V1.3_R4_ASE_LITE(ENG)_v1.0.1.pdf, code: 408\n", - "Downloading targets: 42%|███████████████████████████████████████████████████████████████▏ | 190/457 [01:22<08:11, 1.84s/it]ERROR:sec_certs.sample.certificate:Cert dgst: cert_a9004216663480de_update_f11191787e65fa44 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/0938ma1b_pdf.pdf, code: 408\n", - "ERROR:sec_certs.utils.helpers:Failed to download from https://www.commoncriteriaportal.org/files/epfiles/0915ma3b_pdf.pdf; HTTPSConnectionPool(host='www.commoncriteriaportal.org', port=443): Read timed out.\n", - "ERROR:sec_certs.sample.certificate:Cert dgst: cert_73632a66aceeb90e_update_c6e93ff837a633e2 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/0915ma3b_pdf.pdf, code: nok\n", - "ERROR:sec_certs.sample.certificate:Cert dgst: cert_2a91f42389fda90d_update_475e73ebad1af7a2 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/XSMART%20e-Passport%20V1.4_ASE(BAC%20with%20AA)_ENG_V1.7.pdf, code: 408\n", - "ERROR:sec_certs.utils.helpers:Failed to download from https://www.commoncriteriaportal.org/files/epfiles/0945V2ma2b_pdf.pdf; HTTPSConnectionPool(host='www.commoncriteriaportal.org', port=443): Read timed out.\n", - "ERROR:sec_certs.sample.certificate:Cert dgst: cert_5103648897ba1b3a_update_a05293df88927dad failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/0945V2ma2b_pdf.pdf, code: nok\n", - "ERROR:sec_certs.utils.helpers:Failed to download from https://www.commoncriteriaportal.org/files/epfiles/0915ma1b_pdf.pdf; HTTPSConnectionPool(host='www.commoncriteriaportal.org', port=443): Read timed out.\n", - "ERROR:sec_certs.sample.certificate:Cert dgst: cert_73632a66aceeb90e_update_8ce0c847737c2de3 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/0915ma1b_pdf.pdf, code: nok\n", - "ERROR:sec_certs.utils.helpers:Failed to download from https://www.commoncriteriaportal.org/files/epfiles/anssi_cible_2016_20_lite.pdf; HTTPSConnectionPool(host='www.commoncriteriaportal.org', port=443): Read timed out.\n", - "ERROR:sec_certs.sample.certificate:Cert dgst: cert_fe46772a18258cb2_update_90d86c50dc8c1166 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/anssi_cible_2016_20_lite.pdf, code: nok\n", - "Downloading targets: 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 342/457 [02:44<00:38, 2.99it/s]ERROR:sec_certs.sample.certificate:Cert dgst: cert_5ab17f47ec6820f3_update_0cbfc8dd71545605 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/[ST]%20Samsung%20Multifunction%20MultiXpress%20X4300%20K4350_v1.4(???).pdf, code: nok\n", - "Downloading targets: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 457/457 [03:24<00:00, 2.23it/s]\n", - "Converting reports to txt: 0%| | 0/457 [00:00<?, ?it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_dbaf9789160ca414_update_05b659f9e1f5e5a6.pdf\n", - "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_6785be05d030779b_update_bec315e0f592c9c3.pdf\n", - "Converting reports to txt: 16%|███████████████████████▊ | 74/457 [00:02<00:10, 35.50it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_b891647233f3f188_update_66d87388520b9048.pdf\n", - "Converting reports to txt: 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 410/457 [00:08<00:00, 47.76it/s]ERROR:sec_certs.utils.pdf:Error when converting pdf->txt: poppler error creating document\n", - "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf\n", - "ERROR:sec_certs.utils.pdf:Error during OCR of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf, using garbage: pdftoppm failed: 1\n", + "Downloading reports: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 457/457 [01:12<00:00, 6.33it/s]\n", + "Downloading targets: 74%|████████████████████████████████████████████████████████████████████████▋ | 339/457 [01:08<00:24, 4.89it/s]ERROR:sec_certs.sample.certificate:Cert dgst: cert_5ab17f47ec6820f3_update_0cbfc8dd71545605 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/[ST]%20Samsung%20Multifunction%20MultiXpress%20X4300%20K4350_v1.4(???).pdf, code: nok\n", + "Downloading targets: 92%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 421/457 [01:24<00:06, 5.25it/s]ERROR:sec_certs.sample.certificate:Cert dgst: cert_d07cacdb732c0b0f_update_9dad882431047687 failed to download ST from https://www.commoncriteriaportal.org/files/epfiles/Senetas%20CN%20series%20STv1.1%20Dec%2014.pdf, code: 408\n", + "Downloading targets: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 457/457 [01:32<00:00, 4.95it/s]\n", + "Converting reports to txt: 0%| | 0/457 [00:00<?, ?it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/reports/pdf/cert_6785be05d030779b_update_bec315e0f592c9c3.pdf\n", + "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/reports/pdf/cert_dbaf9789160ca414_update_05b659f9e1f5e5a6.pdf\n", + "Converting reports to txt: 18%|█████████████████ | 84/457 [00:03<00:14, 26.11it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/reports/pdf/cert_b891647233f3f188_update_66d87388520b9048.pdf\n", + "Converting reports to txt: 91%|███████████████████████████████████████████████████████████████████████████████████▎ | 414/457 [00:11<00:01, 38.81it/s]ERROR:sec_certs.utils.pdf:Error when converting pdf->txt: poppler error creating document\n", + "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf\n", + "ERROR:sec_certs.utils.pdf:Error during OCR of cc_04_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf, using garbage: pdftoppm failed: 1\n", "ERROR:sec_certs.sample.certificate:Cert dgst: cert_8ba5d4d02ea73d3d_update_d879046e9be61396 failed to convert report pdf->txt\n", - "Converting reports to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 457/457 [00:21<00:00, 21.21it/s]\n", - "Converting targets to txt: 36%|█████████████████████████████████████████████████████ | 163/448 [00:42<00:42, 6.71it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/targets/pdf/cert_09b43147f1307352_update_a1ff8f3d472941f7.pdf\n", - "Converting targets to txt: 74%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 333/448 [01:21<00:20, 5.51it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/targets/pdf/cert_bd5c7bb29151de01_update_070f8e523f3deba1.pdf\n", - "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/targets/pdf/cert_ef9565620c399b7e_update_b169fd5fee2eb72b.pdf\n", - "Converting targets to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 448/448 [04:58<00:00, 1.50it/s]\n", + "Converting reports to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████| 457/457 [00:26<00:00, 17.33it/s]\n", + "Converting targets to txt: 35%|████████████████████████████████▏ | 159/455 [00:47<00:52, 5.61it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/targets/pdf/cert_09b43147f1307352_update_a1ff8f3d472941f7.pdf\n", + "Converting targets to txt: 75%|████████████████████████████████████████████████████████████████████▉ | 341/455 [01:36<00:31, 3.62it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/targets/pdf/cert_bd5c7bb29151de01_update_070f8e523f3deba1.pdf\n", + "WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/targets/pdf/cert_ef9565620c399b7e_update_b169fd5fee2eb72b.pdf\n", + "Converting targets to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████| 455/455 [05:48<00:00, 1.31it/s]\n", "ERROR:sec_certs.utils.pdf:Error when converting pdf->txt: poppler error creating document\n", - "Converting reports to txt: 0%| | 0/1 [00:00<?, ?it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf\n", - "ERROR:sec_certs.utils.pdf:Error during OCR of /home/jjan/dev/sec-certs/notebooks/cc_03_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf, using garbage: pdftoppm failed: 1\n", + "Converting reports to txt: 0%| | 0/1 [00:00<?, ?it/s]WARNING:sec_certs.utils.pdf:Detected garbage during conversion of cc_04_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf\n", + "ERROR:sec_certs.utils.pdf:Error during OCR of cc_04_10_2022/certs/maintenances/reports/pdf/cert_8ba5d4d02ea73d3d_update_d879046e9be61396.pdf, using garbage: pdftoppm failed: 1\n", "ERROR:sec_certs.sample.certificate:Cert dgst: cert_8ba5d4d02ea73d3d_update_d879046e9be61396 failed to convert report pdf->txt\n", - "Converting reports to txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.00s/it]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Extracting report metadata: 0%| | 0/456 [00:00<?, ?it/s]ERROR:sec_certs.utils.pdf:Failed to read metadata of maintenances/reports/pdf/cert_d455e5408b744b44_update_16f77dbd34439401.pdf, error: PDF metadata unavailable\n", - "Extracting report metadata: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:01<00:00, 452.87it/s]\n", - "Extracting target metadata: 56%|█████████████████████████████████████████████████████████████████████████████████▉ | 253/448 [00:06<00:04, 39.48it/s]ERROR:sec_certs.utils.pdf:Failed to read metadata of maintenances/targets/pdf/cert_bbc41d7d09e40c0c_update_784535cca61b58a7.pdf, error: too many values to unpack (expected 2)\n", - "Extracting target metadata: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 448/448 [00:09<00:00, 49.53it/s]\n", - "Extracting report frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:01<00:00, 453.32it/s]\n", - "Extracting target frontpages: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 448/448 [00:02<00:00, 222.61it/s]\n", - "Extracting report keywords: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:13<00:00, 34.93it/s]\n", - "Extracting target keywords: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 448/448 [03:05<00:00, 2.41it/s]\n" + "Converting reports to txt: 100%|████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.00s/it]\n", + "Extracting report metadata: 0%| | 0/456 [00:00<?, ?it/s]ERROR:sec_certs.utils.pdf:Failed to read metadata of maintenances/reports/pdf/cert_d455e5408b744b44_update_16f77dbd34439401.pdf, error: PDF metadata unavailable\n", + "Extracting report metadata: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:02<00:00, 227.01it/s]\n", + "Extracting target metadata: 64%|██████████████████████████████████████████████████████████▌ | 293/455 [00:09<00:05, 32.35it/s]ERROR:sec_certs.utils.pdf:Failed to read metadata of maintenances/targets/pdf/cert_bbc41d7d09e40c0c_update_784535cca61b58a7.pdf, error: too many values to unpack (expected 2)\n", + "Extracting target metadata: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 455/455 [00:12<00:00, 37.69it/s]\n", + "Extracting report frontpages: 100%|████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:01<00:00, 453.46it/s]\n", + "Extracting target frontpages: 100%|████████████████████████████████████████████████████████████████████████████████████████| 455/455 [00:02<00:00, 226.05it/s]\n", + "Extracting report keywords: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:15<00:00, 30.26it/s]\n", + "Extracting target keywords: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 455/455 [03:55<00:00, 1.94it/s]\n" ] }, { "data": { "text/plain": [ - "<sec_certs.dataset.common_criteria.CCDatasetMaintenanceUpdates at 0x7fcd979bb400>" + "<sec_certs.dataset.common_criteria.CCDatasetMaintenanceUpdates at 0x7fa02b720250>" ] }, - "execution_count": 6, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -768,7 +766,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 14, "metadata": { "pycharm": { "name": "#%%\n" diff --git a/sec_certs/sample/common_criteria.py b/sec_certs/sample/common_criteria.py index 3a9dcc1f..7e1e0abf 100644 --- a/sec_certs/sample/common_criteria.py +++ b/sec_certs/sample/common_criteria.py @@ -661,7 +661,9 @@ class CommonCriteriaCert( for link in list(cell.find_all("a")): if link.get("href") is not None and "/ppfiles/" in link.get("href"): protection_profiles.add( - ProtectionProfile(str(link.contents[0]), CommonCriteriaCert.cc_url + link.get("href")) + ProtectionProfile( + pp_name=str(link.contents[0]), pp_eal=None, pp_link=CommonCriteriaCert.cc_url + link.get("href") + ) ) return protection_profiles |