extraction of lab, lifetime and eal level into processed

3 files changed, 75 insertions, 9 deletions

diff --git a/src/analyze_certificates.py b/src/analyze_certificates.py
index 04ee0b95..cbdaae90 100644
--- a/src/analyze_certificates.py
+++ b/src/analyze_certificates.py
@@ -242,9 +242,22 @@ def analyze_references_graph(filter_rules_group, all_items_found, filter_label):
             if is_in_dict(cert, ['frontpage_scan', 'cert_item']):
                 this_cert_id = cert['processed']['cert_id']
                 if this_cert_id not in certid_info.keys():
+
                     certid_info[this_cert_id] = {}
                 certid_info[this_cert_id]['cert_item'] = cert['frontpage_scan']['cert_item']
 
+    # build cert_id to cert_long_id mapping
+    cert_id_to_long_id_mapping = {}
+    for cert_long_id in all_items_found.keys():
+        cert = all_items_found[cert_long_id]
+        if is_in_dict(cert, ['processed', 'cert_id']):
+            if is_in_dict(cert, ['frontpage_scan', 'cert_item']):
+                this_cert_id = cert['processed']['cert_id']
+                if this_cert_id in cert_id_to_long_id_mapping.keys():
+                    print('ERROR: duplicate cert_id for multiple cert_long_id: {}, {} already used by {}'.format(this_cert_id, cert_long_id, cert_id_to_long_id_mapping[this_cert_id]))
+                else:
+                    cert_id_to_long_id_mapping[this_cert_id] = cert_long_id
+
     # build list of references
     referenced_by = {}
     for cert_long_id in all_items_found.keys():
@@ -281,6 +294,8 @@ def analyze_references_graph(filter_rules_group, all_items_found, filter_label):
     referenced_by_direct_nums = {}
     for cert_id in referenced_by.keys():
         referenced_by_direct_nums[cert_id] = len(referenced_by[cert_id])
+        if cert_id in cert_id_to_long_id_mapping.keys():
+            all_items_found[cert_id_to_long_id_mapping[cert_id]]['processed']['direct_refs'] = len(referenced_by[cert_id])
 
     print('### Certificates sorted by number of other certificates directly referencing them:')
     sorted_ref_direct = sorted(referenced_by_direct_nums.items(), key=operator.itemgetter(1), reverse=False)
@@ -333,6 +348,8 @@ def analyze_references_graph(filter_rules_group, all_items_found, filter_label):
     referenced_by_indirect_nums = {}
     for cert_id in referenced_by_indirect.keys():
         referenced_by_indirect_nums[cert_id] = len(referenced_by_indirect[cert_id])
+        if cert_id in cert_id_to_long_id_mapping.keys():
+            all_items_found[cert_id_to_long_id_mapping[cert_id]]['processed']['indirect_refs'] = referenced_by_indirect_nums[cert_id]
 
     sorted_ref_indirect = sorted(referenced_by_indirect_nums.items(), key=operator.itemgetter(1), reverse=False)
     indirect_refs = []
@@ -385,6 +402,7 @@ def analyze_cert_years_frequency(all_cert_items, filter_label):
     scheme_date = {}
     level_date = {}
     category_date = {}
+    labs_date = {}
     archive_date = {}
     validity_length = {}
     valid_in_years = {}
@@ -420,13 +438,17 @@ def analyze_cert_years_frequency(all_cert_items, filter_label):
             # extract EAL level
             if is_in_dict(cert, ['csv_scan', 'cc_security_level']):
                 level = cert['csv_scan']['cc_security_level']
+                level_split = level.split(",")
                 if level.find(',') != -1:
                     level = level[:level.find(',')]  # trim list of augmented items
-                level_out = level
+                level_out = level_split[0]
                 if level == 'None':
                     if cert['csv_scan']['cc_protection_profiles'] != '':
                         level_out = 'Protection Profile'
 
+                cert['processed']['cc_security_level'] = level_split[0]
+                cert['processed']['cc_security_level_augments'] = level_split[1:]
+
                 if level_out not in level_date.keys():
                     level_date[level_out] = {}
                     for year in range(START_YEAR, END_YEAR):
@@ -465,10 +487,21 @@ def analyze_cert_years_frequency(all_cert_items, filter_label):
                     manufacturer_date[manufacturer][cert_year].append(cert_long_id)
                     manufacturer_items[manufacturer] += 1
 
+            # extract laboratory
+            if is_in_dict(cert, ['processed', 'cert_lab']):
+                lab = cert['processed']['cert_lab']
+
+                if lab not in labs_date.keys():
+                    labs_date[lab] = {}
+                    for year in range(START_YEAR, END_YEAR):
+                        labs_date[lab][year] = []
+                labs_date[lab][cert_year].append(cert_long_id)
+
             # extract cert archival status
             if archived_year is not None:
                 valid_years = archived_year - cert_year + 1
                 validity_length[valid_years].append(cert_long_id)
+                cert['processed']['cert_lifetime_length'] = valid_years
 
                 if 'archived_date' not in archive_date.keys():
                     archive_date['archived_date'] = {}
@@ -510,6 +543,7 @@ def analyze_cert_years_frequency(all_cert_items, filter_label):
     plot_schemes_multi_line_graph(years, scheme_date, ['DE', 'JP', 'FR', 'US', 'CA'], 'Year of issuance', 'Number of certificates issued', fig_label('CC certificates issuance frequency per scheme and year', filter_label), 'num_certs_in_years')
     plot_schemes_multi_line_graph(years, level_date, ['EAL4+', 'EAL5+','EAL2+', 'Protection Profile'], 'Year of issuance', 'Number of certificates issued', fig_label('Certificates issuance frequency per EAL and year', filter_label), 'num_certs_eal_in_years')
     plot_schemes_multi_line_graph(years, category_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Category of certificates issued in given year', filter_label), 'num_certs_category_in_years')
+    plot_schemes_multi_line_graph(years, labs_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Number fo certificates certified by laboratory in given year', filter_label), 'num_certs_by_lab_in_years')
     plot_schemes_multi_line_graph(years_extended, archive_date, [], 'Year of issuance', 'Number of certificates', fig_label('Number of certificates archived or planned for archival in a given year', filter_label), 'num_certs_archived_in_years')
     plot_schemes_multi_line_graph(years_extended, valid_in_years, [], 'Year', 'Number of certificates', fig_label('Number of certificates active and archived in given year', filter_label), 'num_certs_active_archived_in_years')
 
diff --git a/src/extract_certificates.py b/src/extract_certificates.py
index d5011912..13a6de04 100644
--- a/src/extract_certificates.py
+++ b/src/extract_certificates.py
@@ -1760,7 +1760,7 @@ def process_certificates_data(all_cert_items, all_pp_items):
 
     for manuf in sorted_manufacturers:
         # Manufacturer can be single, multiple, separated by - / and ,
-        # heuristics: if separated cnadidate manufacturer can be found in original list (
+        # heuristics: if separated candidate manufacturer can be found in original list (
         # => is sole manufacturer on another certificate => assumption of correct separation)
         separators = [',', '/'] # , '/', ',', 'and']
         multiple_manuf_detected = False
@@ -1856,6 +1856,21 @@ def process_certificates_data(all_cert_items, all_pp_items):
                 cert['processed']['cc_manufacturer_simple_list'] = simple_manufacturers
                 cert['processed']['cc_manufacturer_simple'] = get_manufacturer_simple_name(manufacturer, already_reduced)
 
+        # extract certification lab
+        if is_in_dict(cert, ['frontpage_scan', 'cert_lab']):
+            lab = cert['frontpage_scan']['cert_lab']
+
+            if lab != '':
+                lab = lab.upper()
+                if 'processed' not in cert:
+                    cert['processed'] = {}
+
+                # insert extracted lab - only the first words, changed to uppercase, omitting the rest
+                pos1 = lab.find(' ')
+                if pos1 != -1:
+                    cert['processed']['cert_lab'] = lab[:pos1]
+                else:
+                    cert['processed']['cert_lab'] = lab
     #
     #
     #
diff --git a/src/process_certificates.py b/src/process_certificates.py
index 2fd57047..58a6cee0 100644
--- a/src/process_certificates.py
+++ b/src/process_certificates.py
@@ -37,13 +37,13 @@ def do_analysis_only_smartcards(all_cert_items, current_dir):
 def main():
     # change current directory to store results into results file
     current_dir = os.getcwd()
-    os.chdir(current_dir + '\\..\\results2\\')
+    os.chdir(current_dir + '\\..\\results\\')
 
     cc_html_files_dir = 'c:\\Certs\\web\\'
 
     walk_dir = 'c:\\Certs\\cc_certs_20191208\\cc_certs\\'
     pp_dir = 'c:\\Certs\\cc_certs_20191208\\cc_pp\\'
-    pp_dir = 'c:\\Certs\\cc_certs_20191208\\cc_pp_test3\\'
+    #pp_dir = 'c:\\Certs\\cc_certs_20191208\\cc_pp_test3\\'
     walk_dir_pp = 'c:\\Certs\\pp_20191213\\'
 
     #walk_dir = 'c:\\Certs\\cc_certs_test1\\'
@@ -62,9 +62,11 @@ def main():
     do_extraction = False
     do_extraction_pp = False
     do_pairing = False
+    do_processing = False
     do_analysis = True
 
-    all_pp_front = extract_protectionprofiles_frontpage(pp_dir)
+    #all_pp_front = extract_protectionprofiles_frontpage(pp_dir)
+    #return
 
     if do_extraction:
         all_csv = extract_certificates_csv(cc_html_files_dir)
@@ -103,14 +105,22 @@ def main():
 
         check_expected_cert_results(all_html, all_csv, all_front, all_keywords)
         all_cert_items = collate_certificates_data(all_html, all_csv, all_front, all_keywords, 'link_security_target')
-        all_cert_items = process_certificates_data(all_cert_items, all_pp_items)
         with open("certificate_data_complete.json", "w") as write_file:
             write_file.write(json.dumps(all_cert_items, indent=4, sort_keys=True))
 
-    if do_analysis:
+    if do_processing:
         with open('certificate_data_complete.json') as json_file:
             all_cert_items = json.load(json_file)
-        #all_cert_items = process_certificates_data(all_cert_items)
+        all_pp_items = {}
+
+        all_cert_items = process_certificates_data(all_cert_items, all_pp_items)
+
+        with open("certificate_data_complete_processed.json", "w") as write_file:
+            write_file.write(json.dumps(all_cert_items, indent=4, sort_keys=True))
+
+    if do_analysis:
+        with open('certificate_data_complete_processed.json') as json_file:
+            all_cert_items = json.load(json_file)
 
         current_dir = os.getcwd()
 
@@ -121,9 +131,13 @@ def main():
         # analyze only smartcards
         do_analysis_only_smartcards(all_cert_items, current_dir)
 
+        with open("certificate_data_complete_processed_analyzed.json", "w") as write_file:
+            write_file.write(json.dumps(all_cert_items, indent=4, sort_keys=True))
+
         with open('pp_data_complete.json') as json_file:
             all_pp_items = json.load(json_file)
 
+
     # TODO
     # add extraction of frontpage for protection profiles
     # If None == protection profile => Match PP with its assurance level and recompute
@@ -139,7 +153,10 @@ def main():
     # other schemes: FIPS140-2 certs, EMVCo, Visa Certification, American Express Certification, MasterCard Certification
     # download and analyse CC documentation
     # solve treatment of unicode characters
-
+    # anayze bbliography
+    # histogram/time of cert labs (maybe first two words heuristics)
+    # Statistics about number of characters (length), words, pages
+    # extract frontpage also from other than anssi and bsi certificates (US, BE...)
 
 if __name__ == "__main__":
     main()