added part of pp analysis

1 files changed, 55 insertions, 20 deletions

diff --git a/src/process_certificates.py b/src/process_certificates.py
index ce80651e..d633bc33 100644
--- a/src/process_certificates.py
+++ b/src/process_certificates.py
@@ -5,6 +5,15 @@ import os
 import json
 
 
+def do_all_analysis(all_cert_items, filter_label):
+    analyze_cert_years_frequency(all_cert_items, filter_label)
+    analyze_references_graph(['rules_cert_id'], all_cert_items, filter_label)
+    analyze_eal_frequency(all_cert_items, filter_label)
+    analyze_sars_frequency(all_cert_items, filter_label)
+    generate_dot_graphs(all_cert_items, filter_label)
+    plot_certid_to_item_graph(['keywords_scan', 'rules_protection_profiles'], all_cert_items, filter_label, 'certid_pp_graph.dot', False)
+
+
 def main():
     # change current directory to store results into results file
     current_dir = os.getcwd()
@@ -13,16 +22,27 @@ def main():
     cc_html_files_dir = 'c:\\Certs\\web\\'
 
     walk_dir = 'c:\\Certs\\cc_certs_20191208\\cc_certs\\'
+    pp_dir = 'c:\\Certs\\cc_certs_20191208\\cc_pp\\'
+    #pp_dir = 'c:\\Certs\\cc_certs_20191208\\cc_pp_test1\\'
     walk_dir_pp = 'c:\\Certs\\pp_20191213\\'
 
     #walk_dir = 'c:\\Certs\\cc_certs_test1\\'
     fragments_dir = 'c:\\Certs\\cc_certs_20191208\\cc_certs_txt_fragments\\'
+    pp_fragments_dir = 'c:\\Certs\\cc_certs_20191208\\cc_pp_txt_fragments\\'
 
     generate_basic_download_script()
 
-    do_extraction = False
-    do_extraction_pp = False
-    do_pairing = False
+    all_pp_csv = extract_protectionprofiles_csv(cc_html_files_dir)
+    all_pp_keywords = extract_certificates_keywords(pp_dir, pp_fragments_dir, 'pp')
+    check_expected_pp_results({}, all_pp_csv, {}, all_pp_keywords)
+    all_pp_items = collate_certificates_data({}, all_pp_csv, {}, all_pp_keywords, 'link_pp_document')
+    with open("pp_data_complete.json", "w") as write_file:
+        write_file.write(json.dumps(all_pp_items, indent=4, sort_keys=True))
+
+
+    do_extraction = True
+    do_extraction_pp = True
+    do_pairing = True
     do_analysis = True
 
     if do_extraction:
@@ -31,11 +51,14 @@ def main():
         all_front = extract_certificates_frontpage(walk_dir)
         all_keywords = extract_certificates_keywords(walk_dir, fragments_dir, 'certificate')
 
-    # if do_extraction_pp:
-    #     all_pp_csv = extract_protectionprofiles_csv(cc_html_files_dir)
-    #     all_pp_html = extract_protectionprofiles_html(cc_html_files_dir)
-    #     all_pp_front = extract_protectionprofiles_frontpage(walk_dir_pp)
-    #     all_pp_keywords = extract_certificates_keywords(walk_dir_pp, fragments_dir, 'pp')
+    if do_extraction_pp:
+        all_pp_csv = extract_protectionprofiles_csv(cc_html_files_dir)
+        all_pp_keywords = extract_certificates_keywords(pp_dir, pp_fragments_dir, 'pp')
+        check_expected_pp_results({}, all_pp_csv, {}, all_pp_keywords)
+        all_pp_items = collate_certificates_data({}, all_pp_csv, {}, all_pp_keywords, 'link_pp_document')
+        with open("pp_data_complete.json", "w") as write_file:
+            write_file.write(json.dumps(all_pp_items, indent=4, sort_keys=True))
+
 
     if do_pairing:
         with open('certificate_data_csv_all.json') as json_file:
@@ -57,7 +80,7 @@ def main():
         #     all_pp_keywords = json.load(json_file)
 
         check_expected_cert_results(all_html, all_csv, all_front, all_keywords)
-        all_cert_items = collate_certificates_data(all_html, all_csv, all_front, all_keywords)
+        all_cert_items = collate_certificates_data(all_html, all_csv, all_front, all_keywords, 'link_security_target')
         all_cert_items = process_certificates_data(all_cert_items)
         with open("certificate_data_complete.json", "w") as write_file:
             write_file.write(json.dumps(all_cert_items, indent=4, sort_keys=True))
@@ -70,22 +93,33 @@ def main():
     if do_analysis:
         with open('certificate_data_complete.json') as json_file:
             all_cert_items = json.load(json_file)
-
         #all_cert_items = process_certificates_data(all_cert_items)
 
-        analyze_cert_years_frequency(all_cert_items)
-        analyze_references_graph(['rules_cert_id'], all_cert_items)
-        analyze_eal_frequency(all_cert_items)
-        analyze_sars_frequency(all_cert_items)
-        generate_dot_graphs(all_cert_items, walk_dir)
-        plot_certid_to_item_graph(['keywords_scan', 'rules_protection_profiles'], all_cert_items, walk_dir, 'certid_pp_graph.dot', False)
+        current_dir = os.getcwd()
 
+        # analyze all certificates together
+        os.chdir(current_dir + '\\..\\results\\')
+        do_all_analysis(all_cert_items, '')
 
+        # # archived on 09/01/2019
+        # os.chdir(current_dir + '\\..\\results\\results_archived01092019_only\\')
+        # archived_date = '09/01/2019'
+        # limited_cert_items = {x: all_cert_items[x] for x in all_cert_items if is_in_dict(all_cert_items[x], ['csv_scan', 'cc_archived_date']) and all_cert_items[x]['csv_scan']['cc_archived_date'] == archived_date}
+        # do_all_analysis(limited_cert_items, 'cc_archived_date={}'.format(archived_date))
+        #
+        # # analyze only smartcards
+        # os.chdir(current_dir + '\\..\\results\\results_sc_only\\')
+        # sc_category = 'ICs, Smart Cards and Smart Card-Related Devices and Systems'
+        # sc_cert_items = {x: all_cert_items[x] for x in all_cert_items if is_in_dict(all_cert_items[x], ['csv_scan', 'cc_category']) and all_cert_items[x]['csv_scan']['cc_category'] == sc_category}
+        # print(len(sc_cert_items))
+        # do_all_analysis(sc_cert_items, 'sc_category={}'.format(sc_category))
 
-        # with open('pp_data_complete.json') as json_file:
-        #     all_pp_items = json.load(json_file)
-
+        with open('pp_data_complete.json') as json_file:
+            all_pp_items = json.load(json_file)
 
+    # TODO
+    # add extraction of frontpage for protection profiles
+    # If None == protcetion profile => Match PP with its assurance level and recompute
     #         analyze_sept2019_cleaning(all_cert_items)
     # extract info about protection profiles, download and parse pdf, map to referencing files
     # analysis of PP only: which PP is the most popular?, what schemes/countries are doing most...
@@ -95,7 +129,8 @@ def main():
     # analysis of security targets documents
     # analysis of big cert clusters
     # improve logging (info, warnings, errors, final summary)
-
+    # other schemes: FIPS140-2 certs, EMVCo, Visa Certification, American Express Certification, MasterCard Certification
+    # dlwload and analyse CC documentation ()
 
 if __name__ == "__main__":
     main()