Move downloading to python.

8 files changed, 183 insertions, 352 deletions

diff --git a/sec_certs/__init__.py b/sec_certs/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/sec_certs/__init__.py
diff --git a/sec_certs/analyze_certificates.py b/sec_certs/analyze_certificates.py
index e2e0358c..cce16598 100644
--- a/sec_certs/analyze_certificates.py
+++ b/sec_certs/analyze_certificates.py
@@ -2,6 +2,8 @@ import operator
 import string
 import os
 import datetime
+from pathlib import Path
+
 import numpy as np
 import matplotlib.pyplot as plt
 
@@ -10,8 +12,8 @@ from dateutil import parser
 from graphviz import Digraph
 from tabulate import tabulate
 
-import sanity
-from tags_constants import *
+from . import sanity
+from .tags_constants import *
 
 
 plt.rcdefaults()
@@ -827,3 +829,82 @@ def generate_dot_graphs(all_items_found, filter_label):
     #    print_dot_graph(['rules_protection_profiles'], all_items_found, filter_label, 'rules_protection_profiles.dot', False)
     #    print_dot_graph(['rules_defenses'], all_items_found, filter_label, 'rules_defenses.dot', False)
 
+
+def do_all_analysis(all_cert_items, filter_label):
+    generate_dot_graphs(all_cert_items, filter_label)
+    analyze_cert_years_frequency(all_cert_items, filter_label)
+    analyze_references_graph(['rules_cert_id'], all_cert_items, filter_label)
+    analyze_eal_frequency(all_cert_items, filter_label)
+    analyze_security_assurance_component_frequency(all_cert_items, filter_label)
+    analyze_security_functional_component_frequency(all_cert_items, filter_label)
+    analyze_pdfmeta(all_cert_items, filter_label)
+    plot_certid_to_item_graph(['keywords_scan', 'rules_protection_profiles'], all_cert_items, filter_label, 'certid_pp_graph.dot', False)
+
+
+def do_analysis_everything(all_cert_items, current_dir: Path):
+    if not os.path.exists(current_dir):
+        os.makedirs(current_dir)
+    os.chdir(current_dir)
+    do_all_analysis(all_cert_items, '')
+
+
+def do_analysis_09_01_2019_archival(all_cert_items, current_dir: Path):
+    target_folder = os.path.join(current_dir, 'results_archived01092019_only')
+    if not os.path.exists(target_folder):
+        os.makedirs(target_folder)
+    os.chdir(target_folder)
+    archived_date = '09/01/2019'
+    limited_cert_items = {x: all_cert_items[x] for x in all_cert_items if is_in_dict(all_cert_items[x], ['csv_scan', 'cc_archived_date']) and all_cert_items[x]['csv_scan']['cc_archived_date'] == archived_date}
+    do_all_analysis(limited_cert_items, 'cc_archived_date={}'.format(archived_date))
+
+
+def do_analysis_manufacturers(all_cert_items, current_dir: Path):
+    # analyze only Infineon certificates
+    do_analysis_only_filtered(all_cert_items, current_dir,
+                              ['processed', 'cc_manufacturer_simple'], 'Infineon Technologies AG')
+    # analyze only NXP certificates
+    do_analysis_only_filtered(all_cert_items, current_dir,
+                          ['processed', 'cc_manufacturer_simple'], 'NXP Semiconductors')
+    # analyze only Red Hat certificates
+    do_analysis_only_filtered(all_cert_items, current_dir,
+                              ['processed', 'cc_manufacturer_simple'], 'Red Hat, Inc')
+    # analyze only Suse certificates
+    do_analysis_only_filtered(all_cert_items, current_dir,
+                              ['processed', 'cc_manufacturer_simple'], 'SUSE Linux Products Gmbh')
+
+
+def do_analysis_only_filtered(all_cert_items, current_dir: Path, filter_path, filter_value):
+    filter_string = ''
+    for item in filter_path:
+        if len(filter_string) > 0:
+            filter_string = filter_string + '__'
+        filter_string = filter_string + item
+    target_folder = current_dir / '{}={}'.format(filter_string, filter_value)
+    if not os.path.exists(target_folder):
+        os.makedirs(target_folder)
+    os.chdir(target_folder)
+
+    cert_items = {}
+    for cert_item_key in all_cert_items.keys():
+        item = get_item_from_dict(all_cert_items[cert_item_key], filter_path)
+        if item is not None:
+            if item == filter_value:
+                # Match found, include
+                cert_items[cert_item_key] = all_cert_items[cert_item_key]
+
+    #cert_items = {x: all_cert_items[x] for x in all_cert_items if is_in_dict(all_cert_items[x], ['csv_scan', filter_key]) and all_cert_items[x]['csv_scan'][filter_key] == filter_value}
+
+    print(len(cert_items))
+    do_all_analysis(cert_items, '{}={}'.format(filter_string, filter_value))
+
+
+def do_analysis_only_category(all_cert_items, current_dir: Path, category):
+    do_analysis_only_filtered(all_cert_items, current_dir, ['csv_scan', 'cc_category'], category)
+
+
+def do_analysis_only_smartcards(all_cert_items, current_dir: Path):
+    do_analysis_only_category(all_cert_items, current_dir, 'ICs, Smart Cards and Smart Card-Related Devices and Systems')
+
+
+def do_analysis_only_operatingsystems(all_cert_items, current_dir: Path):
+    do_analysis_only_category(all_cert_items, current_dir, 'Operating Systems')
\ No newline at end of file
diff --git a/sec_certs/download_files.py b/sec_certs/download_files.py
deleted file mode 100644
index d4334b7f..00000000
--- a/sec_certs/download_files.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import extract_certificates
-
-import os
-
-FILE_ERRORS_STRATEGY = extract_certificates.FILE_ERRORS_STRATEGY
-
-
-def generate_fips_basic_download_script():
-    with open('download_fips_web.bat', 'w', errors=FILE_ERRORS_STRATEGY) as file:
-        file.write(
-            'curl "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search'
-            '/all" -o fips_modules_validated.html\n')
-
-
-def generate_fips_download_script(file_name, fips_dir):
-    """generate_fips_download_script.
-
-    :param file_name: name of the download file
-    :param fips_dir: directory for saved files
-    """
-    html_dir = os.path.join(fips_dir, 'html')
-    sp_dir = os.path.join(fips_dir, 'security_policies')
-
-    with open(file_name, 'w', errors=FILE_ERRORS_STRATEGY) as write_file:
-        # make directories for both html and security policies, scraping in one go
-        write_file.write('mkdir {}\n'.format(html_dir))
-        write_file.write('mkdir {}\n\n'.format(sp_dir))
-
-        for cert_id in range(1, 4001):
-            write_file.write(
-                'curl "https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{}" -o {}{}.html\n'.format(
-                    cert_id, html_dir, cert_id))
-            write_file.write(
-                'curl "https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents'
-                '/security-policies/140sp{}.pdf" -o {}{}.pdf\n'.format(
-                    cert_id, sp_dir, cert_id))
-            write_file.write("{} {}{}.pdf\n".format(
-                extract_certificates.PDF2TEXT_CONVERT, sp_dir, cert_id))
diff --git a/sec_certs/extract_certificates.py b/sec_certs/extract_certificates.py
index e2d645c1..302d3185 100644
--- a/sec_certs/extract_certificates.py
+++ b/sec_certs/extract_certificates.py
@@ -3,7 +3,6 @@ import json
 import re
 import os
 import operator
-import string
 
 from enum import Enum
 from pathlib import Path
@@ -11,10 +10,11 @@ import matplotlib.pyplot as plt
 from PyPDF2 import PdfFileReader
 from graphviz import Digraph
 
-import sanity
-from analyze_certificates import is_in_dict
-from cert_rules import rules, fips_rules
-from tags_constants import *
+from . import sanity
+from .analyze_certificates import is_in_dict
+from .cert_rules import rules, fips_rules
+from .files import search_files, load_cert_html_file, FILE_ERRORS_STRATEGY
+from .tags_constants import *
 
 plt.rcdefaults()
 
@@ -22,10 +22,6 @@ plt.rcdefaults()
 # Used as sanity check during development to detect sudden drop in number of extracted features
 APPEND_DETAILED_MATCH_MATCHES = False
 VERBOSE = False
-FILE_ERRORS_STRATEGY = 'surrogateescape'
-'replace'
-# FILE_ERRORS_STRATEGY = 'strict'
-CC_WEB_URL = 'https://www.commoncriteriaportal.org'
 PDF2TEXT_CONVERT = 'pdftotext -raw'
 
 REGEXEC_SEP = '[ ,;\]”)(]'
@@ -33,11 +29,6 @@ LINE_SEPARATOR = ' '
 # LINE_SEPARATOR = ''  # if newline is not replaced with space, long string included in matches are found
 
 
-def search_files(folder):
-    for root, dirs, files in os.walk(folder):
-        yield from [os.path.join(root, x) for x in files]
-
-
 def get_line_number(lines, line_length_compensation, match_start_index):
     line_chars_offset = 0
     line_number = 1
@@ -93,20 +84,6 @@ def load_cert_file(file_name, limit_max_lines=-1, line_separator=LINE_SEPARATOR)
     return whole_text, whole_text_with_newlines, was_unicode_decode_error
 
 
-def load_cert_html_file(file_name):
-    with open(file_name, 'r', errors=FILE_ERRORS_STRATEGY) as f:
-        try:
-            whole_text = f.read()
-        except UnicodeDecodeError:
-            f.close()
-            with open(file_name, "r", encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
-                try:
-                    whole_text = f2.read()
-                except UnicodeDecodeError:
-                    print('### ERROR: failed to read file {}'.format(file_name))
-    return whole_text
-
-
 def normalize_match_string(match):
     # normalize match
     match = match.strip()
@@ -346,18 +323,6 @@ def print_specified_property_sorted(section_name, item_name, items_found_all):
         print(item)
 
 
-def print_found_properties(items_found_all):
-    print_specified_property_sorted(TAG_CERT_ID, items_found_all)
-    print_specified_property_sorted(TAG_CERT_ITEM, items_found_all)
-    print_specified_property_sorted(TAG_CERT_ITEM_VERSION, items_found_all)
-    print_specified_property_sorted(
-        TAG_REFERENCED_PROTECTION_PROFILES, items_found_all)
-    print_specified_property_sorted(TAG_CC_VERSION, items_found_all)
-    print_specified_property_sorted(TAG_CC_SECURITY_LEVEL, items_found_all)
-    print_specified_property_sorted(TAG_DEVELOPER, items_found_all)
-    print_specified_property_sorted(TAG_CERT_LAB, items_found_all)
-
-
 def search_only_headers_bsi(walk_dir: Path):
     print('BSI HEADER SEARCH')
     LINE_SEPARATOR_STRICT = ' '
@@ -466,9 +431,6 @@ def search_only_headers_bsi(walk_dir: Path):
         if no_match_yet:
             files_without_match.append(file_name)
 
-    if False:
-        print_found_properties(items_found_all)
-
     print('\n*** Certificates without detected preface:')
     for file_name in files_without_match:
         print('No hits for {}'.format(file_name))
@@ -679,9 +641,6 @@ def search_only_headers_anssi(walk_dir: Path):
         if no_match_yet:
             files_without_match.append(file_name)
 
-    if False:
-        print_found_properties(items_found_all)
-
     print('\n*** Certificates without detected preface:')
     for file_name in files_without_match:
         print('No hits for {}'.format(file_name))
@@ -1062,9 +1021,6 @@ def search_pp_only_headers(walk_dir: Path):
         if no_match_yet:
             files_without_match.append(file_name)
 
-    if False:
-        print_found_properties(items_found_all)
-
     print('\n*** Protection profiles without detected header:')
     for file_name in files_without_match:
         print('No hits for {}'.format(file_name))
@@ -1748,76 +1704,25 @@ def extract_pp_metadata_csv(file_name):
     return items_found_all, download_files_certs, download_files_maintainance
 
 
-def generate_download_script(file_name, certs_dir, targets_dir, base_url, download_files_certs):
-    with open(file_name, "w", errors=FILE_ERRORS_STRATEGY) as write_file:
-        # certs files
-        if certs_dir != '':
-            write_file.write('mkdir \"{}\"\n'.format(certs_dir))
-            write_file.write('cd \"{}\"\n\n'.format(certs_dir))
-        for cert in download_files_certs:
-            # double %% is necessary to prevent replacement of %2 within script (second argument of script)
-            file_name_short_web = cert[0].replace(' ', '%%20')
-
-            if file_name_short_web.find(base_url) != -1:
-                # base url already included
-                write_file.write(
-                    'curl \"{}\" -o \"{}\"\n'.format(file_name_short_web, cert[1]))
-            else:
-                # insert base url
-                write_file.write(
-                    'curl \"{}{}\" -o \"{}\"\n'.format(base_url, file_name_short_web, cert[1]))
-            write_file.write('{} \"{}\"\n\n'.format(PDF2TEXT_CONVERT, cert[1]))
-
-        if len(download_files_certs) > 0 and len(cert) > 2:
-            # security targets file
-            if targets_dir != '':
-                write_file.write('\n\ncd ..\n')
-                write_file.write('mkdir \"{}\"\n'.format(targets_dir))
-                write_file.write('cd \"{}\"\n\n'.format(targets_dir))
-            for cert in download_files_certs:
-                # double %% is necessary to prevent replacement of %2 within script (second argument of script)
-                file_name_short_web = cert[2].replace(' ', '%%20')
-                if file_name_short_web.find(base_url) != -1:
-                    # base url already included
-                    write_file.write(
-                        'curl \"{}\" -o \"{}\"\n'.format(file_name_short_web, cert[3]))
-                else:
-                    # insert base url
-                    write_file.write(
-                        'curl \"{}{}\" -o \"{}\"\n'.format(base_url, file_name_short_web, cert[3]))
-                write_file.write('{} \"{}\"\n\n'.format(
-                    PDF2TEXT_CONVERT, cert[3]))
-
-
 def extract_certificates_html(web_dir: Path):
     file_name = web_dir / 'cc_products_active.html'
-    items_found_all_active, download_files_certs, download_files_updates = extract_certificates_metadata_html(
+    items_found_all_active, certs_active, updates_active = extract_certificates_metadata_html(
         file_name)
     for item in items_found_all_active.keys():
         items_found_all_active[item]['html_scan']['cert_status'] = 'active'
 
-    generate_download_script('download_active_certs.bat',
-                             'certs', 'targets', CC_WEB_URL, download_files_certs)
-    generate_download_script('download_active_updates.bat',
-                             'certs', 'targets', CC_WEB_URL, download_files_updates)
-
     file_name = web_dir / 'cc_products_archived.html'
-    items_found_all_archived, download_files_certs, download_files_updates = extract_certificates_metadata_html(
+    items_found_all_archived, certs_archive, updates_archive = extract_certificates_metadata_html(
         file_name)
     for item in items_found_all_archived.keys():
         items_found_all_archived[item]['html_scan']['cert_status'] = 'archived'
 
-    generate_download_script('download_archived_certs.bat',
-                             'certs', 'targets', CC_WEB_URL, download_files_certs)
-    generate_download_script('download_archived_updates.bat',
-                             'certs', 'targets', CC_WEB_URL, download_files_updates)
-
     items_found_all = {**items_found_all_active, **items_found_all_archived}
 
-    return items_found_all
+    return items_found_all, certs_active + certs_archive, updates_active + updates_archive
 
 
-def extract_certificates_csv(web_dir: Path, results_dir: Path):
+def extract_certificates_csv(web_dir: Path):
     file_name = web_dir / 'cc_products_active.csv'
     items_found_all_active = extract_certificates_metadata_csv(file_name)
     for item in items_found_all_active.keys():
@@ -1840,10 +1745,6 @@ def extract_protectionprofiles_csv(base_dir: Path):
     for item in items_found_all_active.keys():
         items_found_all_active[item]['csv_scan']['cert_status'] = 'active'
 
-    generate_download_script('download_active_pp.bat',
-                             'pp_report', 'pp', CC_WEB_URL, download_files_pp)
-    generate_download_script('download_active_pp_updates.bat',
-                             'pp_updates', '', CC_WEB_URL, download_files_pp_updates)
 
     file_name = base_dir / 'cc_pp_archived.csv'
     items_found_all_archived, download_files_pp, download_files_pp_updates = extract_pp_metadata_csv(
@@ -1851,11 +1752,6 @@ def extract_protectionprofiles_csv(base_dir: Path):
     for item in items_found_all_archived.keys():
         items_found_all_archived[item]['csv_scan']['cert_status'] = 'archived'
 
-    generate_download_script('download_archived_pp.bat',
-                             'pp_report', 'pp', CC_WEB_URL, download_files_pp)
-    generate_download_script('download_archived_pp_updates.bat',
-                             'pp_updates', '', CC_WEB_URL, download_files_pp_updates)
-
     items_found_all = {**items_found_all_active, **items_found_all_archived}
 
     return items_found_all
@@ -2303,71 +2199,3 @@ def process_certificates_data(all_cert_items, all_pp_items):
                             cert['processed']['pp_filename'] = sub_profile['pp_filename']
 
     return all_cert_items
-
-
-def generate_basic_download_script(web_dir: Path):
-    with open(web_dir / 'download_cc_web.bat', 'w', errors=FILE_ERRORS_STRATEGY) as file:
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/products/\" -o cc_products_active.html\n')
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/products/index.cfm?archived=1\" -o cc_products_archived.html\n\n')
-
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/labs/\" -o cc_labs.html\n')
-
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/products/certified_products.csv\" -o cc_products_active.csv\n')
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/products/certified_products-archived.csv\" -o cc_products_archived.csv\n\n')
-
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/pps/\" -o cc_pp_active.html\n')
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/pps/collaborativePP.cfm?cpp=1\" -o cc_pp_collaborative.html\n')
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/pps/index.cfm?archived=1\" -o cc_pp_archived.html\n\n')
-
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/pps/pps.csv\" -o cc_pp_active.csv\n')
-        file.write(
-            'curl \"https://www.commoncriteriaportal.org/pps/pps-archived.csv\" -o cc_pp_archived.csv\n\n')
-
-
-def generate_failed_download_script(base_dir: Path):
-    # obtain list of all downloaded pdf files and their size
-    # check for pdf files with too small length
-    # generate download script again (single one)
-
-    # visit all relevant subfolders
-    sub_folders = ['active/certs', 'active/targets', 'active_update/certs', 'active_update/targets',
-                   'archived/certs', 'archived/targets', 'archived_update/certs', 'archived_update/targets']
-
-    # the smallest correct certificate downloaded was 71kB, if server error occurred, it was only 1245 bytes
-    MIN_CORRECT_CERT_SIZE = 5000
-    download_again = []
-    for sub_folder in sub_folders:
-        target_dir = base_dir / sub_folder
-        # obtain list of all downloaded pdf files and their size
-        files = search_files(target_dir)
-        for file_name in files:
-            # process only .pdf files
-            if not os.path.isfile(file_name):
-                continue
-            file_ext = file_name[file_name.rfind('.'):].upper()
-            if file_ext != '.PDF' and file_ext != '.DOC' and file_ext != '.DOCX':
-                continue
-
-            # obtain size of file
-            file_size = os.path.getsize(file_name)
-            if file_size < MIN_CORRECT_CERT_SIZE:
-                # too small file, likely failed download - retry
-                file_name_short = file_name[file_name.rfind(os.sep) + 1:]
-                # double %% is necessary to prevent replacement of %2 within script (second argument of script)
-                file_name_short_web = file_name_short.replace(' ', '%%20')
-                download_link = '/files/epfiles/{}'.format(file_name_short_web)
-                download_again.append((download_link, file_name))
-
-    generate_download_script('download_failed_certs.bat',
-                             '', '', CC_WEB_URL, download_again)
-    print('*** Number of files to be re-downloaded again (inside \'{}\'): {}'.format(
-        'download_failed_certs.bat', len(download_again)))
diff --git a/sec_certs/files.py b/sec_certs/files.py
new file mode 100644
index 00000000..dcd3262f
--- /dev/null
+++ b/sec_certs/files.py
@@ -0,0 +1,36 @@
+import json
+import os
+
+FILE_ERRORS_STRATEGY = 'surrogateescape'
+'replace'
+# FILE_ERRORS_STRATEGY = 'strict'
+
+def search_files(folder):
+    for root, dirs, files in os.walk(folder):
+        yield from [os.path.join(root, x) for x in files]
+
+
+def load_cert_html_file(file_name):
+    with open(file_name, 'r', errors=FILE_ERRORS_STRATEGY) as f:
+        try:
+            whole_text = f.read()
+        except UnicodeDecodeError:
+            f.close()
+            with open(file_name, "r", encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2:
+                try:
+                    whole_text = f2.read()
+                except UnicodeDecodeError:
+                    print('### ERROR: failed to read file {}'.format(file_name))
+    return whole_text
+
+
+def load_json_files(files_list):
+    loaded_jsons = []
+    for file_name in files_list:
+        with open(file_name) as json_file:
+            loaded_items = json.load(json_file)
+            loaded_jsons.append(loaded_items)
+            print('{} loaded, total items = {}'.format(file_name, len(loaded_items)))
+    return tuple(loaded_jsons)
+
+
diff --git a/sec_certs/fips_certificates.py b/sec_certs/fips_certificates.py
index 60d2d6ed..31e43f55 100755
--- a/sec_certs/fips_certificates.py
+++ b/sec_certs/fips_certificates.py
@@ -17,7 +17,9 @@ import extract_certificates
 from process_certificates import load_json_files
 from cert_rules import rules_fips_htmls as RE_FIPS_HTMLS
 
-FILE_ERRORS_STRATEGY = extract_certificates.FILE_ERRORS_STRATEGY
+import sec_certs.files
+
+FILE_ERRORS_STRATEGY = sec_certs.files.FILE_ERRORS_STRATEGY
 FIPS_BASE_URL = 'https://csrc.nist.gov'
 FIPS_MODULE_URL = 'https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/'
 
@@ -96,10 +98,10 @@ def fips_search_html(base_dir, output_file, dump_to_file=False):
 
     all_found_items = {}
 
-    for file in extract_certificates.search_files(base_dir):
+    for file in sec_certs.files.search_files(base_dir):
         items_found = {}
         initialize_entry(items_found)
-        text = extract_certificates.load_cert_html_file(file)
+        text = sec_certs.files.load_cert_html_file(file)
         filename = os.path.splitext(os.path.basename(file))[0]
         all_found_items[filename] = items_found
         items_found['cert_fips_id'] = filename
@@ -423,7 +425,7 @@ def main(directory):
     items, html = load_json_files(files_to_load)
 
     print("FINDING TABLES")
-    not_decoded = extract_certs_from_tables(extract_certificates.search_files(policies_dir), html)
+    not_decoded = extract_certs_from_tables(sec_certs.files.search_files(policies_dir), html)
 
     print("NOT DECODED:", not_decoded)
     with open(results_dir / 'broken_files.json', 'w') as f:
diff --git a/sec_certs/process_certificates.py b/sec_certs/process_certificates.py
index 00be17a0..f248c420 100755
--- a/sec_certs/process_certificates.py
+++ b/sec_certs/process_certificates.py
@@ -1,117 +1,33 @@
 #!/usr/bin/env python3
-import os
-import json
-from pathlib import Path
 
 import click
 
-from extract_certificates import *
-from analyze_certificates import *
-
-
-def do_all_analysis(all_cert_items, filter_label):
-    generate_dot_graphs(all_cert_items, filter_label)
-    analyze_cert_years_frequency(all_cert_items, filter_label)
-    analyze_references_graph(['rules_cert_id'], all_cert_items, filter_label)
-    analyze_eal_frequency(all_cert_items, filter_label)
-    analyze_security_assurance_component_frequency(all_cert_items, filter_label)
-    analyze_security_functional_component_frequency(all_cert_items, filter_label)
-    analyze_pdfmeta(all_cert_items, filter_label)
-    plot_certid_to_item_graph(['keywords_scan', 'rules_protection_profiles'], all_cert_items, filter_label, 'certid_pp_graph.dot', False)
-
-
-def do_analysis_everything(all_cert_items, current_dir: Path):
-    if not os.path.exists(current_dir):
-        os.makedirs(current_dir)
-    os.chdir(current_dir)
-    do_all_analysis(all_cert_items, '')
-
-
-def do_analysis_09_01_2019_archival(all_cert_items, current_dir: Path):
-    target_folder = os.path.join(current_dir, 'results_archived01092019_only')
-    if not os.path.exists(target_folder):
-        os.makedirs(target_folder)
-    os.chdir(target_folder)
-    archived_date = '09/01/2019'
-    limited_cert_items = {x: all_cert_items[x] for x in all_cert_items if is_in_dict(all_cert_items[x], ['csv_scan', 'cc_archived_date']) and all_cert_items[x]['csv_scan']['cc_archived_date'] == archived_date}
-    do_all_analysis(limited_cert_items, 'cc_archived_date={}'.format(archived_date))
-
-
-def do_analysis_manufacturers(all_cert_items, current_dir: Path):
-    # analyze only Infineon certificates
-    do_analysis_only_filtered(all_cert_items, current_dir,
-                              ['processed', 'cc_manufacturer_simple'], 'Infineon Technologies AG')
-    # analyze only NXP certificates
-    do_analysis_only_filtered(all_cert_items, current_dir,
-                          ['processed', 'cc_manufacturer_simple'], 'NXP Semiconductors')
-    # analyze only Red Hat certificates
-    do_analysis_only_filtered(all_cert_items, current_dir,
-                              ['processed', 'cc_manufacturer_simple'], 'Red Hat, Inc')
-    # analyze only Suse certificates
-    do_analysis_only_filtered(all_cert_items, current_dir,
-                              ['processed', 'cc_manufacturer_simple'], 'SUSE Linux Products Gmbh')
-
-
-def do_analysis_only_filtered(all_cert_items, current_dir: Path, filter_path, filter_value):
-    filter_string = ''
-    for item in filter_path:
-        if len(filter_string) > 0:
-            filter_string = filter_string + '__'
-        filter_string = filter_string + item
-    target_folder = current_dir / '{}={}'.format(filter_string, filter_value)
-    if not os.path.exists(target_folder):
-        os.makedirs(target_folder)
-    os.chdir(target_folder)
-
-    cert_items = {}
-    for cert_item_key in all_cert_items.keys():
-        item = get_item_from_dict(all_cert_items[cert_item_key], filter_path)
-        if item is not None:
-            if item == filter_value:
-                # Match found, include
-                cert_items[cert_item_key] = all_cert_items[cert_item_key]
-
-    #cert_items = {x: all_cert_items[x] for x in all_cert_items if is_in_dict(all_cert_items[x], ['csv_scan', filter_key]) and all_cert_items[x]['csv_scan'][filter_key] == filter_value}
-
-    print(len(cert_items))
-    do_all_analysis(cert_items, '{}={}'.format(filter_string, filter_value))
-
-
-def do_analysis_only_category(all_cert_items, current_dir: Path, category):
-    do_analysis_only_filtered(all_cert_items, current_dir, ['csv_scan', 'cc_category'], category)
-
-
-def do_analysis_only_smartcards(all_cert_items, current_dir: Path):
-    do_analysis_only_category(all_cert_items, current_dir, 'ICs, Smart Cards and Smart Card-Related Devices and Systems')
-
-
-def do_analysis_only_operatingsystems(all_cert_items, current_dir: Path):
-    do_analysis_only_category(all_cert_items, current_dir, 'Operating Systems')
-
-
-def load_json_files(files_list):
-    loaded_jsons = []
-    for file_name in files_list:
-        with open(file_name) as json_file:
-            loaded_items = json.load(json_file)
-            loaded_jsons.append(loaded_items)
-            print('{} loaded, total items = {}'.format(file_name, len(loaded_items)))
-    return tuple(loaded_jsons)
+from sec_certs.files import load_json_files
+from .extract_certificates import *
+from .analyze_certificates import *
+from .download import download_cc_web, download_cc
 
 
 @click.command()
 @click.argument("directory", required=True, type=str)
 @click.option("--fresh", "do_complete_extraction", is_flag=True, help="Whether to extract from a fresh state.")
-@click.option("--do-download", "do_download_certs", is_flag=True, help="Whether to download certificate pages.")
-@click.option("--do-extraction", "do_extraction", is_flag=True, help="Whether to extract information from the certs.")
+@click.option("--do-download-meta", "do_download_meta", is_flag=True, help="Whether to download meta pages.")
+@click.option("--do-extraction-meta", "do_extraction_meta", is_flag=True, help="Whether to extract information from the meta pages.")
+@click.option("--do-download-certs", "do_download_certs", is_flag=True, help="Whether to download certs.")
+@click.option("--do-extraction", "do_extraction_certs", is_flag=True, help="Whether to extract information from the certs.")
 @click.option("--do-pairing", "do_pairing", is_flag=True, help="Whether to pair PP stuff.")
 @click.option("--do-processing", "do_processing", is_flag=True, help="Whether to process certificates.")
-@click.option("--do-anaysis", "do_analysis", is_flag=True, help="Whether to analyse certificates.")
-def main(directory, do_complete_extraction: bool, do_download_certs: bool, do_extraction: bool, do_pairing: bool, do_processing: bool, do_analysis: bool):
+@click.option("--do-analysis", "do_analysis", is_flag=True, help="Whether to analyse certificates.")
+@click.option("-t", "--threads", "threads", type=int, default=4, help="Amount of threads to use.")
+def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_extraction_meta: bool,
+         do_download_certs: bool, do_extraction_certs: bool,
+         do_pairing: bool, do_processing: bool, do_analysis: bool, threads: int):
     directory = Path(directory)
 
     web_dir = directory / "web"
     walk_dir = directory / "certs"
+    certs_dir = walk_dir / "certs"
+    targets_dir = walk_dir / "targets"
     pp_dir = directory / "pp"
     fragments_dir = directory / "cert_fragments"
     pp_fragments_dir = directory / "pp_fragments"
@@ -119,45 +35,38 @@ def main(directory, do_complete_extraction: bool, do_download_certs: bool, do_ex
 
     web_dir.mkdir(parents=True, exist_ok=True)
     walk_dir.mkdir(parents=True, exist_ok=True)
+    certs_dir.mkdir(parents=True, exist_ok=True)
+    targets_dir.mkdir(parents=True, exist_ok=True)
     pp_dir.mkdir(parents=True, exist_ok=True)
     fragments_dir.mkdir(parents=True, exist_ok=True)
     pp_fragments_dir.mkdir(parents=True, exist_ok=True)
     results_dir.mkdir(parents=True, exist_ok=True)
 
-    # 1. generate_basic_download_script
-    # 2. run and download basic cc files from webpage (no certs yet)
-
     #
     # Start processing
     #
-    generate_basic_download_script(web_dir)
-    generate_failed_download_script(walk_dir)
-
-    #do_complete_extraction = True
-    #do_download_certs = True
-    #do_extraction = True
-    #do_pairing = True
-    #do_processing = True
-    #do_analysis = True
     do_analysis_filtered = False
 
     if do_complete_extraction:
         # analyze all files from scratch, set 'previous' state to empty dict
         prev_csv = {}
         prev_html = {}
+        prev_download = []
         prev_front = {}
         prev_keywords = {}
         prev_pdf_meta = {}
     else:
         # load previously analyzed results
-        prev_csv, prev_html, prev_front, prev_keywords, prev_pdf_meta = load_json_files(
-            map(lambda x: results_dir / x, ['certificate_data_csv_all.json', 'certificate_data_html_all.json', 'certificate_data_frontpage_all.json',
-             'certificate_data_keywords_all.json', 'certificate_data_pdfmeta_all.json']))
+        prev_csv, prev_html, prev_download, prev_front, prev_keywords, prev_pdf_meta = load_json_files(
+            map(lambda x: results_dir / x, ['certificate_data_csv_all.json',
+                                            'certificate_data_html_all.json',
+                                            'certificate_data_download_all.json',
+                                            'certificate_data_frontpage_all.json',
+                                            'certificate_data_keywords_all.json',
+                                            'certificate_data_pdfmeta_all.json']))
 
-    if do_download_certs:
-        # extract_certificates_html() will generate download scripts for cert documents
-        # NOTE: download scripts must be run manually now
-        current_html = extract_certificates_html(web_dir)
+    if do_download_meta:
+        download_cc_web(web_dir)
 
         # NOTE: Code below is preparation for differetian download of only new certificates
         # - unfinished now
@@ -185,18 +94,27 @@ def main(directory, do_complete_extraction: bool, do_download_certs: bool, do_ex
         #
         # print('*** New items detected: {}'.format(len(new_items)))
 
-    if do_extraction:
+    if do_extraction_meta:
         all_csv = extract_certificates_csv(web_dir)
-        all_html = extract_certificates_html(web_dir)
+        all_html, certs, updates = extract_certificates_html(web_dir)
+
+        with open(results_dir / "certificate_data_csv_all.json", "w") as write_file:
+            json.dump(all_csv, write_file, indent=4, sort_keys=True)
+        with open(results_dir / "certificate_data_html_all.json", "w") as write_file:
+            json.dump(all_html, write_file, indent=4, sort_keys=True)
+        with open(results_dir / "certificate_data_download_all.json", "w") as write_file:
+            json.dump(certs + updates, write_file, indent=4, sort_keys=True)
+
+    if do_download_certs:
+        all_download = load_json_files([results_dir / "certificate_data_download_all.json"])
+        download_cc(walk_dir, all_download[0], threads)
+
+    if do_extraction_certs:
         all_front = extract_certificates_frontpage(walk_dir)
         all_keywords = extract_certificates_keywords(walk_dir, fragments_dir, 'certificate')
         all_pdf_meta = extract_certificates_pdfmeta(walk_dir, 'certificate', results_dir)
 
         # save joined results
-        with open(results_dir / "certificate_data_csv_all.json", "w") as write_file:
-            json.dump(all_csv, write_file, indent=4, sort_keys=True)
-        with open(results_dir / "certificate_data_html_all.json", "w") as write_file:
-            json.dump(all_html,  write_file, indent=4, sort_keys=True)
         with open(results_dir / "certificate_data_frontpage_all.json", "w") as write_file:
             json.dump(all_front,  write_file, indent=4, sort_keys=True)
         with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file:
@@ -237,8 +155,11 @@ def main(directory, do_complete_extraction: bool, do_download_certs: bool, do_ex
         # CERTIFICATES
         # load results from previous step
         all_csv, all_html, all_front, all_keywords, all_pdf_meta = load_json_files(
-            map(lambda x: results_dir / x, ['certificate_data_csv_all.json', 'certificate_data_html_all.json', 'certificate_data_frontpage_all.json',
-             'certificate_data_keywords_all.json', 'certificate_data_pdfmeta_all.json']))
+            map(lambda x: results_dir / x, ['certificate_data_csv_all.json',
+                                            'certificate_data_html_all.json',
+                                            'certificate_data_frontpage_all.json',
+                                            'certificate_data_keywords_all.json',
+                                            'certificate_data_pdfmeta_all.json']))
         # check for unexpected results
         check_expected_cert_results(all_html, all_csv, all_front, all_keywords, all_pdf_meta)
         # collate all results into single file
diff --git a/setup.py b/setup.py
index 124a3957..1cb455c5 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,8 @@ setup(
         "tabulate",
         "tabula-py",
         "pikepdf",
-        "Click"
+        "Click",
+        "requests"
     ],
     entry_points = """
         [console_scripts]