From f4b41527535a204b47b1aeecd3e28cfb49fb8b5b Mon Sep 17 00:00:00 2001 From: Petr Svenda Date: Fri, 6 Dec 2019 16:58:04 +0100 Subject: processing inter-certs references --- src/cert_rules.py | 1 + src/search_certficate.py | 97 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/cert_rules.py b/src/cert_rules.py index 4afd2e9e..fa182697 100644 --- a/src/cert_rules.py +++ b/src/cert_rules.py @@ -138,6 +138,7 @@ rules_crypto_libs = [ 'AT1 Secure RSA/ECC/SHA library', 'Crypto Library [v]*[0-9.]+', 'ATMEL Toolbox [0-9.]+', + 'v1.02.013' # Infineon's ROCA-vulnerable library ] diff --git a/src/search_certficate.py b/src/search_certficate.py index ec55a24c..29d157ce 100644 --- a/src/search_certficate.py +++ b/src/search_certficate.py @@ -12,6 +12,7 @@ from collections import defaultdict STOP_ON_INCORRECT_NUMS = False APPEND_DETAILED_MATCH_MATCHES = False +VERBOSE = False REGEXEC_SEP = '[ ,;\]”)(]' LINE_SEPARATOR = ' ' @@ -295,6 +296,8 @@ def print_dot_graph(filter_rules_group, all_items_found, walk_dir, out_dot_name, this_cert_id = '' if defaultdict(lambda: defaultdict(lambda: None), cert)['processed']['cert_id'] is not None: this_cert_id = cert['processed']['cert_id'] + if defaultdict(lambda: defaultdict(lambda: None), cert)['html_scan']['cert_item_name'] is not None: + this_cert_name = cert['html_scan']['cert_item_name'] just_file_name = cert['html_scan']['link_cert_report_file_name'] @@ -326,7 +329,7 @@ def print_dot_graph(filter_rules_group, all_items_found, walk_dir, out_dot_name, print('{} pdf rendered'.format(out_dot_name)) -def analyze_graph(filter_rules_group, all_items_found): +def analyze_references_graph(filter_rules_group, all_items_found): # build cert_id to item name mapping certid_info = {} for cert_long_id in all_items_found.keys(): @@ -338,7 +341,7 @@ def analyze_graph(filter_rules_group, all_items_found): certid_info[this_cert_id] = {} certid_info[this_cert_id]['cert_item'] = cert['frontpage_scan']['cert_item'] - + # build list of references referenced_by = {} for cert_long_id in all_items_found.keys(): # do not continue if no keywords were extracted () @@ -368,9 +371,9 @@ def analyze_graph(filter_rules_group, all_items_found): if this_cert_id not in referenced_by[match]: referenced_by[match].append(this_cert_id) - # process referenced_by - + # # process direct references + # referenced_by_direct_nums = {} for cert_id in referenced_by.keys(): referenced_by_direct_nums[cert_id] = len(referenced_by[cert_id]) @@ -379,11 +382,53 @@ def analyze_graph(filter_rules_group, all_items_found): sorted_ref_direct = sorted(referenced_by_direct_nums.items(), key=operator.itemgetter(1), reverse=False) for cert_id in sorted_ref_direct: if defaultdict(lambda: defaultdict(lambda: None), certid_info)[cert_id[0]]['cert_item'] is not None: - print(' {} : {}x : {}'.format(cert_id[0], cert_id[1], certid_info[cert_id[0]]['cert_item'])) + print(' {} : {}x directly: {}'.format(cert_id[0], cert_id[1], certid_info[cert_id[0]]['cert_item'])) else: - print(' {} : {}x'.format(cert_id[0], cert_id[1])) + print(' {} : {}x directly'.format(cert_id[0], cert_id[1])) print(' Total number of certificates referenced at least once: {}'.format(len(sorted_ref_direct))) + EXPECTED_CERTS_REFERENCED_ONCE = 937 + if EXPECTED_CERTS_REFERENCED_ONCE != len(sorted_ref_direct): + print(' ERROR: Different than expected num certificates referenced at least once: {} vs. {}'.format(EXPECTED_CERTS_REFERENCED_ONCE, len(sorted_ref_direct))) + if STOP_ON_INCORRECT_NUMS: + print(error_different_than_expected) + + # + # compute indirect num of references + # + referenced_by_indirect = {} + for cert_id in referenced_by.keys(): + referenced_by_indirect[cert_id] = set() + for item in referenced_by[cert_id]: + referenced_by_indirect[cert_id].add(item) + + new_change_detected = True + while new_change_detected: + new_change_detected = False + + certids_list = referenced_by.keys() + for cert_id in certids_list: + tmp_referenced_by_indirect_nums = referenced_by_indirect[cert_id].copy() + for referencing in tmp_referenced_by_indirect_nums: + if referencing in referenced_by.keys(): + tmp_referencing = referenced_by_indirect[referencing].copy() + for in_referencing in tmp_referencing: + if in_referencing not in referenced_by_indirect[cert_id]: + new_change_detected = True + referenced_by_indirect[cert_id].add(in_referencing) + + print('### Certificates sorted by number of other certificates indirectly referencing them:') + referenced_by_indirect_nums = {} + for cert_id in referenced_by_indirect.keys(): + referenced_by_indirect_nums[cert_id] = len(referenced_by_indirect[cert_id]) + + sorted_ref_indirect = sorted(referenced_by_indirect_nums.items(), key=operator.itemgetter(1), reverse=False) + for cert_id in sorted_ref_indirect: + if defaultdict(lambda: defaultdict(lambda: None), certid_info)[cert_id[0]]['cert_item'] is not None: + print(' {} : {}x indirectly: {}'.format(cert_id[0], cert_id[1], certid_info[cert_id[0]]['cert_item'])) + else: + print(' {} : {}x indirectly'.format(cert_id[0], cert_id[1])) + def estimate_cert_id(frontpage_scan, keywords_scan, file_name): # check if cert id was extracted from frontpage (most priority) @@ -405,7 +450,8 @@ def estimate_cert_id(frontpage_scan, keywords_scan, file_name): max_occurences = num_occurences keywords_cert_id = match num_items_found_certid_group += num_occurences - print(' -> most frequent cert id: {}, {}x'.format(keywords_cert_id, num_items_found_certid_group)) + if VERBOSE: + print(' -> most frequent cert id: {}, {}x'.format(keywords_cert_id, num_items_found_certid_group)) # try to search for certificate id directly in file name - if found, higher priority filename_cert_id = '' @@ -420,10 +466,11 @@ def estimate_cert_id(frontpage_scan, keywords_scan, file_name): print(' -> cert id found directly in certificate name: {}'.format(matches[0])) filename_cert_id = matches[0] - print('Identified cert ids for {}:'.format(file_name)) - print(' frontpage_cert_id: {}'.format(frontpage_cert_id)) - print(' filename_cert_id: {}'.format(filename_cert_id)) - print(' keywords_cert_id: {}'.format(keywords_cert_id)) + if VERBOSE: + print('Identified cert ids for {}:'.format(file_name)) + print(' frontpage_cert_id: {}'.format(frontpage_cert_id)) + print(' filename_cert_id: {}'.format(filename_cert_id)) + print(' keywords_cert_id: {}'.format(keywords_cert_id)) if frontpage_cert_id != '': return frontpage_cert_id @@ -572,10 +619,6 @@ def search_only_headers_bsi(walk_dir): items_found[TAG_CC_VERSION] = normalize_match_string(cc_version) items_found[TAG_CC_SECURITY_LEVEL] = normalize_match_string(cc_security_level) - # we now have full raw header extracted - try to process raw data - # items_found_all[file_name][TAG_CERT_HEADER_PROCESSED] = {} - # items_found_all[file_name][TAG_CERT_HEADER_PROCESSED] = process_raw_header(items_found) - if no_match_yet: files_without_match.append(file_name) @@ -669,6 +712,13 @@ def search_only_headers_anssi(walk_dir): whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(file_name) + # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs + pos = whole_text.find(' ') + if pos != -1: + pos = whole_text.find(' ', pos) + if pos != -1: + whole_text = whole_text[pos:] + no_match_yet = True other_rule_already_match = False other_rule = '' @@ -843,6 +893,7 @@ def extract_certificates_keywords(walk_dir, fragments_dir): def parse_product_updates(updates_chunk, link_files_updates): maintenance_reports = [] + rule_with_maintainance_ST = '.*?([0-9]+?-[0-9]+?-[0-9]+?) (.+?)\