Add basic CLI options (directory and processing step).

3 files changed, 65 insertions, 84 deletions

diff --git a/sec_certs/extract_certificates.py b/sec_certs/extract_certificates.py
index ed09e41a..cc0bc8af 100644
--- a/sec_certs/extract_certificates.py
+++ b/sec_certs/extract_certificates.py
@@ -6,6 +6,7 @@ import operator
 import string
 
 from enum import Enum
+from pathlib import Path
 import matplotlib.pyplot as plt
 from PyPDF2 import PdfFileReader
 from graphviz import Digraph
@@ -359,7 +360,7 @@ def print_found_properties(items_found_all):
     print_specified_property_sorted(TAG_CERT_LAB, items_found_all)
 
 
-def search_only_headers_bsi(walk_dir):
+def search_only_headers_bsi(walk_dir: Path):
     print('BSI HEADER SEARCH')
     LINE_SEPARATOR_STRICT = ' '
     NUM_LINES_TO_INVESTIGATE = 15
@@ -482,7 +483,7 @@ def search_only_headers_bsi(walk_dir):
     return items_found_all, files_without_match
 
 
-def search_only_headers_anssi(walk_dir):
+def search_only_headers_anssi(walk_dir: Path):
     class HEADER_TYPE(Enum):
         HEADER_FULL = 1
         HEADER_MISSING_CERT_ITEM_VERSION = 2
@@ -709,7 +710,7 @@ def search_only_headers_anssi(walk_dir):
     return items_found_all, files_without_match
 
 
-def extract_certificates_frontpage(walk_dir, write_output_file=True):
+def extract_certificates_frontpage(walk_dir: Path, write_output_file=True):
     anssi_items_found, anssi_files_without_match = search_only_headers_anssi(
         walk_dir)
     bsi_items_found, bsi_files_without_match = search_only_headers_bsi(
@@ -733,7 +734,7 @@ def extract_certificates_frontpage(walk_dir, write_output_file=True):
     return items_found_all
 
 
-def search_pp_only_headers(walk_dir):
+def search_pp_only_headers(walk_dir: Path):
     # LINE_SEPARATOR_STRICT = ' '
     # NUM_LINES_TO_INVESTIGATE = 15
     # rules_certificate_preface = [
@@ -1090,7 +1091,7 @@ def search_pp_only_headers(walk_dir):
     return items_found_all, files_without_match
 
 
-def extract_protectionprofiles_frontpage(walk_dir, write_output_file=True):
+def extract_protectionprofiles_frontpage(walk_dir: Path, write_output_file=True):
     pp_items_found, pp_files_without_match = search_pp_only_headers(walk_dir)
 
     print('*** Files without detected protection profiles header')
@@ -1107,7 +1108,7 @@ def extract_protectionprofiles_frontpage(walk_dir, write_output_file=True):
     return pp_items_found
 
 
-def extract_certificates_keywords(walk_dir, fragments_dir, file_prefix, write_output_file=True,
+def extract_certificates_keywords(walk_dir: Path, fragments_dir: Path, file_prefix, write_output_file=True,
                                   should_censure_right_away=False, fips_items=None):
     # ensure existence of fragments folder
     if not os.path.exists(fragments_dir):
@@ -1183,7 +1184,7 @@ def extract_certificates_keywords(walk_dir, fragments_dir, file_prefix, write_ou
     return all_items_found
 
 
-def extract_certificates_pdfmeta(walk_dir, file_prefix, write_output_file=True):
+def extract_certificates_pdfmeta(walk_dir: Path, file_prefix, write_output_file=True):
     all_items_found = {}
     counter = 0
     for file_name in search_files(walk_dir):
@@ -1825,8 +1826,8 @@ def generate_download_script(file_name, certs_dir, targets_dir, base_url, downlo
                     PDF2TEXT_CONVERT, cert[3]))
 
 
-def extract_certificates_html(base_dir, write_output_file=True):
-    file_name = '{}cc_products_active.html'.format(base_dir)
+def extract_certificates_html(base_dir: Path, write_output_file: bool = True):
+    file_name = base_dir / 'cc_products_active.html'
     items_found_all_active, download_files_certs, download_files_updates = extract_certificates_metadata_html(
         file_name)
     for item in items_found_all_active.keys():
@@ -1842,7 +1843,7 @@ def extract_certificates_html(base_dir, write_output_file=True):
     generate_download_script('download_active_updates.bat',
                              'certs', 'targets', CC_WEB_URL, download_files_updates)
 
-    file_name = '{}cc_products_archived.html'.format(base_dir)
+    file_name = base_dir / 'cc_products_archived.html'
     items_found_all_archived, download_files_certs, download_files_updates = extract_certificates_metadata_html(
         file_name)
     for item in items_found_all_archived.keys():
@@ -1868,13 +1869,13 @@ def extract_certificates_html(base_dir, write_output_file=True):
     return items_found_all
 
 
-def extract_certificates_csv(base_dir, write_output_file=True):
-    file_name = '{}cc_products_active.csv'.format(base_dir)
+def extract_certificates_csv(base_dir: Path, write_output_file: bool = True):
+    file_name = base_dir / 'cc_products_active.csv'
     items_found_all_active = extract_certificates_metadata_csv(file_name)
     for item in items_found_all_active.keys():
         items_found_all_active[item]['csv_scan']['cert_status'] = 'active'
 
-    file_name = '{}cc_products_archived.csv'.format(base_dir)
+    file_name =  base_dir / 'cc_products_archived.csv'
     items_found_all_archived = extract_certificates_metadata_csv(file_name)
     for item in items_found_all_archived.keys():
         items_found_all_archived[item]['csv_scan']['cert_status'] = 'archived'
@@ -1889,8 +1890,8 @@ def extract_certificates_csv(base_dir, write_output_file=True):
     return items_found_all
 
 
-def extract_protectionprofiles_csv(base_dir, write_output_file=True):
-    file_name = '{}cc_pp_active.csv'.format(base_dir)
+def extract_protectionprofiles_csv(base_dir: Path, write_output_file: bool = True):
+    file_name = base_dir / 'cc_pp_active.csv'
     items_found_all_active, download_files_pp, download_files_pp_updates = extract_pp_metadata_csv(
         file_name)
     for item in items_found_all_active.keys():
@@ -1901,7 +1902,7 @@ def extract_protectionprofiles_csv(base_dir, write_output_file=True):
     generate_download_script('download_active_pp_updates.bat',
                              'pp_updates', '', CC_WEB_URL, download_files_pp_updates)
 
-    file_name = '{}cc_pp_archived.csv'.format(base_dir)
+    file_name = base_dir / 'cc_pp_archived.csv'
     items_found_all_archived, download_files_pp, download_files_pp_updates = extract_pp_metadata_csv(
         file_name)
     for item in items_found_all_archived.keys():
@@ -2366,8 +2367,8 @@ def process_certificates_data(all_cert_items, all_pp_items):
     return all_cert_items
 
 
-def generate_basic_download_script():
-    with open('download_cc_web.bat', 'w', errors=FILE_ERRORS_STRATEGY) as file:
+def generate_basic_download_script(web_dir: Path):
+    with open(web_dir / 'download_cc_web.bat', 'w', errors=FILE_ERRORS_STRATEGY) as file:
         file.write(
             'curl \"https://www.commoncriteriaportal.org/products/\" -o cc_products_active.html\n')
         file.write(
@@ -2394,7 +2395,7 @@ def generate_basic_download_script():
             'curl \"https://www.commoncriteriaportal.org/pps/pps-archived.csv\" -o cc_pp_archived.csv\n\n')
 
 
-def generate_failed_download_script(base_dir):
+def generate_failed_download_script(base_dir: Path):
     # obtain list of all downloaded pdf files and their size
     # check for pdf files with too small length
     # generate download script again (single one)
@@ -2407,7 +2408,7 @@ def generate_failed_download_script(base_dir):
     MIN_CORRECT_CERT_SIZE = 5000
     download_again = []
     for sub_folder in sub_folders:
-        target_dir = os.path.join(base_dir, sub_folder)
+        target_dir = base_dir / sub_folder
         # obtain list of all downloaded pdf files and their size
         files = search_files(target_dir)
         for file_name in files:
diff --git a/sec_certs/process_certificates.py b/sec_certs/process_certificates.py
index 3612eafe..41f44247 100755
--- a/sec_certs/process_certificates.py
+++ b/sec_certs/process_certificates.py
@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 import os
 import json
+from pathlib import Path
+
+import click
 
 from extract_certificates import *
 from analyze_certificates import *
@@ -101,69 +104,45 @@ def sanitize_all_strings(data):
 
     if isinstance(data, dict):
         for k, v in data.items():
-            if isinstance(v, dict) or isinstance(v, list) or isinstance(v, tuple):
+            if isinstance(v, (dict, list, tuple)):
                 sanitize_all_strings(v)
             elif isinstance(v, str):
                 sanitized = ''.join(filter(lambda x: x in printable, v))
                 data[k] = ''.join(filter(lambda x: x in printable, v))
 
-    if isinstance(data, list) or isinstance(data, tuple):
+    if isinstance(data, (list, tuple)):
         for v in data:
-            if isinstance(v, dict) or isinstance(v, list) or isinstance(v, tuple):
+            if isinstance(v, (dict, list, tuple)):
                 sanitize_all_strings(v)
             elif isinstance(v, str):
                 sanitized = ''.join(filter(lambda x: x in printable, v))
                 v = ''.join(filter(lambda x: x in printable, v))
 
 
-def main():
-    # Paths for certificates downloaded on 20191208
-    paths_20191208 = {}
-    paths_20191208['id'] = '20191208'
-    paths_20191208['cc_web_files_dir'] = 'c:\\Certs\\cc_certs_20191208\\web\\'
-    paths_20191208['walk_dir'] = 'c:\\Certs\\cc_certs_20191208\\cc_certs\\'
-    #paths_20191208['walk_dir'] = 'c:\\Certs\\cc_certs_20191208\\cc_certs_test1\\'
-    paths_20191208['pp_dir'] = 'c:\\Certs\\cc_certs_20191208\\cc_pp\\'
-    #paths_20191208['pp_dir'] = 'c:\\Certs\\cc_certs_20191208\\cc_pp_test1\\'
-    paths_20191208['fragments_dir'] = 'c:\\Certs\\cc_certs_20191208\\cc_certs_txt_fragments\\'
-    paths_20191208['pp_fragments_dir'] = 'c:\\Certs\\cc_certs_20191208\\cc_pp_txt_fragments\\'
-
-    # Paths for certificates downloaded on 20200225
-    paths_20200225 = {}
-    paths_20200225['id'] = '20200225'
-    paths_20200225['cc_web_files_dir'] = 'c:\\Certs\\cc_certs_20200225\\web\\'
-    paths_20200225['walk_dir'] = 'c:\\Certs\\cc_certs_20200225\\cc_certs\\'
-    paths_20200225['pp_dir'] = 'c:\\Certs\\cc_certs_20200225\\cc_pp\\'
-    paths_20200225['fragments_dir'] = 'c:\\Certs\\cc_certs_20200225\\cc_certs_txt_fragments\\'
-    paths_20200225['pp_fragments_dir'] = 'c:\\Certs\\cc_certs_20200225\\cc_pp_txt_fragments\\'
-
-    # Paths for certificates downloaded on 20200904
-    paths_20200904 = {}
-    paths_20200904['id'] = '20200904'
-    paths_20200904['cc_web_files_dir'] = 'c:\\Certs\\cc_certs_20200904\\web\\'
-    paths_20200904['cc_pp_web_files_dir'] = 'c:\\Certs\\certs_pp_20201008\\pp_web\\'
-    paths_20200904['walk_dir'] = 'c:\\Certs\\cc_certs_20200904\\cc_certs\\'
-    paths_20200904['pp_dir'] = 'c:\\Certs\\certs_pp_20201008\\cc_pp\\'
-    paths_20200904['fragments_dir'] = 'c:\\Certs\\cc_certs_20200904\\cc_certs_txt_fragments\\'
-    paths_20200904['pp_fragments_dir'] = 'c:\\Certs\\certs_pp_20201008\\cc_pp_txt_fragments\\'
-
-    # initialize paths based on the profile used
-    #paths_used = paths_20191208
-    #paths_used = paths_20200225
-    paths_used = paths_20200904
-    #paths_used['id'] = 'temp' # change id for temporary debugging
+@click.command()
+@click.option("-d", "--dir", "directory", type=str, help="The directory to use.", required=True)
+@click.option("--fresh", "do_complete_extraction", is_flag=True, help="Whether to extract from a fresh state.")
+@click.option("--do-download", "do_download_certs", is_flag=True, help="Whether to download certificate pages.")
+@click.option("--do-extraction", "do_extraction", is_flag=True, help="Whether to extract information from the certs.")
+@click.option("--do-pairing", "do_pairing", is_flag=True, help="Whether to pair PP stuff.")
+@click.option("--do-processing", "do_processing", is_flag=True, help="Whether to process certificates.")
+@click.option("--do-anaysis", "do_analysis", is_flag=True, help="Whether to analyse certificates.")
+def main(directory, do_complete_extraction, do_download_certs, do_extraction, do_pairing, do_processing, do_analysis):
+    directory = Path(directory)
 
-    cc_web_files_dir = paths_used['cc_web_files_dir']
-    walk_dir = paths_used['walk_dir']
-    fragments_dir = paths_used['fragments_dir']
+    web_dir = directory / "web"
+    walk_dir = directory / "certs"
+    pp_dir = directory / "pp"
+    fragments_dir = directory / "cert_fragments"
+    pp_fragments_dir = directory / "pp_fragments"
+    results_folder = directory / "results"
 
-    # results folder includes unique identification of input dataset
-    results_folder = '{}\\..\\results_{}\\'.format(os.getcwd(), paths_used['id'])
-    # ensure existence of results folder
-    if not os.path.exists(results_folder):
-        os.makedirs(results_folder)
-    # change current directory to store results into results file
-    os.chdir(results_folder)
+    web_dir.mkdir(parents=True, exist_ok=True)
+    walk_dir.mkdir(parents=True, exist_ok=True)
+    pp_dir.mkdir(parents=True, exist_ok=True)
+    fragments_dir.mkdir(parents=True, exist_ok=True)
+    pp_fragments_dir.mkdir(parents=True, exist_ok=True)
+    results_folder.mkdir(parents=True, exist_ok=True)
 
     # 1. generate_basic_download_script
     # 2. run and download basic cc files from webpage (no certs yet)
@@ -171,16 +150,16 @@ def main():
     #
     # Start processing
     #
-    generate_basic_download_script()
+    generate_basic_download_script(web_dir)
     generate_failed_download_script(walk_dir)
 
-    do_complete_extraction = True
-    do_download_certs = True
-    do_extraction = True
-    do_pairing = True
-    do_processing = True
-    do_analysis = True
-    do_analysis_filtered = True
+    #do_complete_extraction = True
+    #do_download_certs = True
+    #do_extraction = True
+    #do_pairing = True
+    #do_processing = True
+    #do_analysis = True
+    do_analysis_filtered = False
 
     if do_complete_extraction:
         # analyze all files from scratch, set 'previous' state to empty dict
@@ -198,7 +177,7 @@ def main():
     if do_download_certs:
         # extract_certificates_html() will generate download scripts for cert documents
         # NOTE: download scripts must be run manually now
-        current_html = extract_certificates_html(cc_web_files_dir, False)
+        current_html = extract_certificates_html(web_dir, False)
 
         # NOTE: Code below is preparation for differetian download of only new certificates
         # - unfinished now
@@ -227,8 +206,8 @@ def main():
         # print('*** New items detected: {}'.format(len(new_items)))
 
     if do_extraction:
-        all_csv = extract_certificates_csv(cc_web_files_dir, False)
-        all_html = extract_certificates_html(cc_web_files_dir, False)
+        all_csv = extract_certificates_csv(web_dir, False)
+        all_html = extract_certificates_html(web_dir, False)
         all_front = extract_certificates_frontpage(walk_dir, False)
         all_keywords = extract_certificates_keywords(walk_dir, fragments_dir, 'certificate', False)
         all_pdf_meta = extract_certificates_pdfmeta(walk_dir, 'certificate', False)
diff --git a/setup.py b/setup.py
index 4f59a739..9d1469ba 100644
--- a/setup.py
+++ b/setup.py
@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(
     name='sec-certs',
     author='Petr Svenda',
-    author_email='',
+    author_email='svenda@fi.muni.cz',
     version='0.0.0',
-    packages=['sec_certs'],
+    packages=find_packages(),
     license='MIT',
     description="Tool for analysis of security certificates",
     long_description=open("README.md").read(),
@@ -27,6 +27,7 @@ setup(
         "numpy",
         "tabulate",
         "tabula-py",
-        "pikepdf"
+        "pikepdf",
+        "Click"
     ]
 )