From ddc875e5c43c2ba15d5149ed55fdf3f55ff20df1 Mon Sep 17 00:00:00 2001 From: Petr Svenda Date: Mon, 11 Jan 2021 11:35:56 +0100 Subject: add early saving of extracted data --- process_certificates.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/process_certificates.py b/process_certificates.py index 96d9591a..2b63d70a 100755 --- a/process_certificates.py +++ b/process_certificates.py @@ -114,19 +114,18 @@ def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_ext if do_extraction_certs: all_keywords = extract_certificates_keywords_parallel(walk_dir, fragments_dir, 'certificate', threads) - #all_keywords = extract_certificates_keywords(walk_dir, fragments_dir, 'certificate') - all_front = extract_certificates_frontpage(walk_dir) - all_pdf_meta = extract_certificates_pdfmeta_parallel(walk_dir, 'certificate', threads) - #all_pdf_meta = extract_certificates_pdfmeta(walk_dir, 'certificate', results_dir) + with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file: + json.dump(all_keywords, write_file, indent=4, sort_keys=True) - # save joined results + all_front = extract_certificates_frontpage(walk_dir) with open(results_dir / "certificate_data_frontpage_all.json", "w") as write_file: json.dump(all_front, write_file, indent=4, sort_keys=True) - with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file: - json.dump(all_keywords, write_file, indent=4, sort_keys=True) + + all_pdf_meta = extract_certificates_pdfmeta_parallel(walk_dir, 'certificate', threads) with open(results_dir / "certificate_data_pdfmeta_all.json", "w") as write_file: json.dump(all_pdf_meta, write_file, indent=4, sort_keys=True) + # if do_extraction_pp: # all_pp_csv = extract_protectionprofiles_csv(web_dir) # all_pp_front = extract_protectionprofiles_frontpage(pp_dir) -- cgit v1.3.1