added extraction of pdf metadata

2 files changed, 50 insertions, 0 deletions

diff --git a/src/extract_certificates.py b/src/extract_certificates.py
index 13a6de04..990647f9 100644
--- a/src/extract_certificates.py
+++ b/src/extract_certificates.py
@@ -11,6 +11,8 @@ from enum import Enum
 import matplotlib.pyplot as plt; plt.rcdefaults()
 from tags_constants import *
 
+from PyPDF2 import PdfFileReader
+
 # if True, then exception is raised when unexpect intermediate number is obtained
 # Used as sanity check during development to detect sudden drop in number of extracted features
 STOP_ON_UNEXPECTED_NUMS = False
@@ -970,6 +972,50 @@ def extract_certificates_keywords(walk_dir, fragments_dir, file_prefix):
     return all_items_found
 
 
+
+def extract_certificates_pdfmeta(walk_dir, file_prefix):
+    all_items_found = {}
+    for file_name in search_files(walk_dir):
+        if not os.path.isfile(file_name):
+            continue
+        file_ext = file_name[file_name.rfind('.'):]
+        if file_ext != '.pdf':
+            continue
+
+        print('*** {} ***'.format(file_name))
+
+        item = {}
+
+        item['pdf_file_size_bytes'] = os.path.getsize(file_name)
+        try:
+            with open(file_name, 'rb') as f:
+                pdf = PdfFileReader(f)
+
+                info = pdf.getDocumentInfo()
+                if info is not None:
+                    for key in info:
+                        item[key] = info[key]
+                    # item['pdf_author'] = info['/Author']
+                    # item['pdf_creator'] = info.creator
+                    # item['pdf_producer'] = info.producer
+                    # item['pdf_subject'] = info.subject
+                    # item['pdf_title'] = info.title
+                    # item['pdf_creation_date'] = info.author_raw
+
+                item['pdf_is_encrypted'] = pdf.getIsEncrypted()
+                item['pdf_number_of_pages'] = pdf.getNumPages()
+        except Exception as e:
+            item['error'] = str(e)
+
+        all_items_found[file_name] = item
+
+    # store results into file with fixed name and also with time appendix
+    with open("{}_data_pdfmeta_all.json".format(file_prefix), "w") as write_file:
+        write_file.write(json.dumps(all_items_found, indent=4, sort_keys=True))
+
+    return all_items_found
+
+
 def extract_file_name_from_url(url):
     file_name = url[url.rfind('/') + 1:]
     file_name = file_name.replace('%20', ' ')
diff --git a/src/process_certificates.py b/src/process_certificates.py
index 58a6cee0..b074fa51 100644
--- a/src/process_certificates.py
+++ b/src/process_certificates.py
@@ -67,12 +67,14 @@ def main():
 
     #all_pp_front = extract_protectionprofiles_frontpage(pp_dir)
     #return
+    all_pdf_meta = extract_certificates_pdfmeta(walk_dir, 'certificate')
 
     if do_extraction:
         all_csv = extract_certificates_csv(cc_html_files_dir)
         all_html = extract_certificates_html(cc_html_files_dir)
         all_front = extract_certificates_frontpage(walk_dir)
         all_keywords = extract_certificates_keywords(walk_dir, fragments_dir, 'certificate')
+        all_pdf_meta = extract_certificates_pdfmeta(walk_dir, 'certificate')
 
     if do_extraction_pp:
         all_pp_csv = extract_protectionprofiles_csv(cc_html_files_dir)
@@ -139,6 +141,8 @@ def main():
 
 
     # TODO
+    # extract pdf file metadata (pdf header, file size...), PyPDF2 https://www.blog.pythonlibrary.org/2018/04/10/extracting-pdf-metadata-and-text-with-python/, https://github.com/pdfminer/pdfminer.six
+    #   https://pythonhosted.org/PyPDF2/PdfFileReader.html
     # add extraction of frontpage for protection profiles
     # If None == protection profile => Match PP with its assurance level and recompute
     #         analyze_sept2019_cleaning(all_cert_items)