From 64bf9c880828fa36f8625fdabad468668df62575 Mon Sep 17 00:00:00 2001 From: J08nY Date: Thu, 15 Oct 2020 21:56:24 +0200 Subject: Add a proper setup, move into a package. --- src/analyze_certificates.py | 828 --------------- src/cert_rules.py | 357 ------- src/download_files.py | 36 - src/extract_certificates.py | 2435 ------------------------------------------- src/fips_certificates.py | 444 -------- src/process_certificates.py | 377 ------- src/sanity.py | 53 - src/tags_constants.py | 27 - 8 files changed, 4557 deletions(-) delete mode 100644 src/analyze_certificates.py delete mode 100644 src/cert_rules.py delete mode 100644 src/download_files.py delete mode 100644 src/extract_certificates.py delete mode 100644 src/fips_certificates.py delete mode 100644 src/process_certificates.py delete mode 100644 src/sanity.py delete mode 100644 src/tags_constants.py (limited to 'src') diff --git a/src/analyze_certificates.py b/src/analyze_certificates.py deleted file mode 100644 index cc0beba4..00000000 --- a/src/analyze_certificates.py +++ /dev/null @@ -1,828 +0,0 @@ -import operator -from graphviz import Digraph -from tabulate import tabulate -import matplotlib.pyplot as plt; - -import sanity - -plt.rcdefaults() -import numpy as np -import matplotlib.pyplot as plt -from matplotlib.pyplot import figure -from dateutil import parser -import datetime -from tags_constants import * -import string -import os - -STOP_ON_UNEXPECTED_NUMS = False - -printable = set(string.printable) - - -def is_in_dict(target_dict, path): - current_level = target_dict - for item in path: - if item not in current_level: - return False - else: - current_level = current_level[item] - return True - - -def get_item_from_dict(target_dict, path): - current_level = target_dict - for item in path: - if item not in current_level: - return None - else: - current_level = current_level[item] - return current_level - - -def fig_label(title, filter): - if filter != '': - return '{}\nfilter: {}'.format(title, filter) - else: - return title - - -def plot_bar_graph(data, x_data_labels, y_label, title, file_name): - fig_width = round(len(data) / 2) - if fig_width < 10: - fig_width = 10 - figure(num=None, figsize=(fig_width, 8), dpi=200, facecolor='w', edgecolor='k') - y_pos = np.arange(len(x_data_labels)) - plt.bar(y_pos, data, align='center', alpha=0.5) - plt.xticks(y_pos, x_data_labels) - plt.xticks(rotation=45) - plt.ylabel(y_label) - plt.title(title) - x1, x2, y1, y2 = plt.axis() - plt.axis((x1, x2, y1 - 1, y2)) - plt.savefig(file_name + '.png', bbox_inches='tight') - plt.savefig(file_name + '.pdf', bbox_inches='tight') - plt.close() - - -def plot_heatmap_graph(data_matrix, x_data_ticks, y_data_ticks, x_label, y_label, title, file_name): - fig_size = round(len(x_data_ticks) / 2) - if fig_size == 0: - fig_size = 8 - plt.figure(figsize=(fig_size, 8), dpi=200, facecolor='w', edgecolor='k') - #color_map = 'BuGn' - color_map = 'Purples' - plt.imshow(data_matrix, cmap=color_map, interpolation='none', aspect='auto') - #sns.heatmap(data_matrix, cmap=color_map, linewidth=0.5) - x_pos = np.arange(len(y_data_ticks)) - plt.yticks(x_pos, y_data_ticks) - y_pos = np.arange(len(x_data_ticks)) - plt.xticks(y_pos, x_data_ticks) - plt.xticks(rotation=90, ha='center') - plt.gca().invert_yaxis() - x1, x2, y1, y2 = plt.axis() - plt.axis((x1, x2, y1 - 0.5, y2)) - plt.xlabel(x_label) - plt.ylabel(y_label) - plt.title(title) - try: - plt.savefig(file_name + '.png', bbox_inches='tight') - except RuntimeError as e: - print('RuntimeError while writing {} file as png'.format(file_name + '.png')) - try: - plt.savefig(file_name + '.pdf', bbox_inches='tight') - except RuntimeError as e: - print('RuntimeError while writing {} file as pdf'.format(file_name + '.pdf')) - plt.close() - - -def compute_and_plot_hist(data, bins, y_label, title, file_name): - hist_refs = np.histogram(data, bins) - hist_labels = [] - for index in range(0, len(bins) - 1): - if bins[index] == bins[index + 1] - 1: - hist_labels.append('{}'.format(bins[index])) - else: - hist_labels.append('{}-{}'.format(bins[index], bins[index + 1])) - # plot bar graph with number of certificates referenced by given number of other certificates - plot_bar_graph(hist_refs[0], hist_labels, y_label, title, file_name) - - -def depricated_print_dot_graph_keywordsonly(filter_rules_group, all_items_found, cert_id, filter_label, out_dot_name, thick_as_occurences): - # print dot - dot = Digraph(comment='Certificate ecosystem: {}'.format(filter_rules_group)) - dot.attr('graph', label='{}'.format(filter_label), labelloc='t', fontsize='30') - dot.attr('node', style='filled') - - # insert nodes believed to be cert id for the processed certificates - for cert in cert_id.keys(): - if cert != "": - dot.attr('node', color='green') - dot.node(cert_id[cert]) - - dot.attr('node', color='gray') - for file_name in all_items_found.keys(): - just_file_name = file_name - this_cert_id = cert_id[file_name] - - if file_name.rfind(os.sep) != -1: - just_file_name = file_name[file_name.rfind(os.sep) + 1:] - - # insert file name and identified probable certification id - if this_cert_id != "": - dot.edge(this_cert_id, this_cert_id, label=just_file_name) - - items_found_group = all_items_found[file_name] - for rules_group in items_found_group.keys(): - - # process only specified rule groups - if rules_group not in filter_rules_group: - continue - - items_found = items_found_group[rules_group] - for rule in items_found.keys(): - for match in items_found[rule]: - if match != this_cert_id: - if thick_as_occurences: - num_occurrences = str(items_found[rule][match][TAG_MATCH_COUNTER] / 3 + 1) - else: - num_occurrences = '1' - label = str(items_found[rule][match][TAG_MATCH_COUNTER]) # label with number of occurrences - if this_cert_id != "": - dot.edge(this_cert_id, match, color='orange', style='solid', label=label, penwidth=num_occurrences) - - # Generate dot graph using GraphViz into pdf - dot.render(out_dot_name, view=False) - print('{} pdf rendered'.format(out_dot_name)) - - -def get_cert_node_label(cert_item, print_item_name): - if print_item_name: - sanitized_name = ''.join(filter(lambda x: x in printable, cert_item['csv_scan']['cert_item_name'])) - #sanitized_name = cert_item['csv_scan']['cert_item_name'].encode('ascii', 'ignore') - #sanitized_name = cert_item['csv_scan']['cert_item_name'].encode("ascii") - sanitized_name = sanitized_name.replace(':', ' ') # ':' is not allowed in dot - sanitized_name = sanitized_name.replace('/', ' ') # '/' is not allowed in dot - return '{}\n{}'.format(cert_item['processed']['cert_id'], sanitized_name) - else: - return cert_item['processed']['cert_id'] - - -def print_dot_graph(filter_rules_group, all_items_found, filter_label, out_dot_name, thick_as_occurences, print_item_name): - # print dot - dot = Digraph(comment='Certificate ecosystem: {}'.format(filter_rules_group)) - dot.attr('graph', label='{}'.format(filter_label), labelloc='t', fontsize='30') - dot.attr('node', style='filled') - - # insert nodes believed to be cert id for the processed certificates - cert_id_to_long_id_map = {} - for cert_long_id in all_items_found.keys(): - if is_in_dict(all_items_found[cert_long_id], ['processed', 'cert_id']): - dot.attr('node', color='green') # URL='https://www.commoncriteriaportal.org/' doesn't work for pdf - this_cert_node_label = get_cert_node_label(all_items_found[cert_long_id], print_item_name) - # basic node id is cert id, but possibly add additional info - dot.node(all_items_found[cert_long_id]['processed']['cert_id'], label=this_cert_node_label) - cert_id_to_long_id_map[all_items_found[cert_long_id]['processed']['cert_id']] = cert_long_id - - dot.attr('node', color='gray') - for cert_long_id in all_items_found.keys(): - # do not continue if no keywords were extracted - if 'keywords_scan' not in all_items_found[cert_long_id].keys(): - continue - - cert = all_items_found[cert_long_id] - this_cert_id = '' - if is_in_dict(cert, ['processed', 'cert_id']): - this_cert_id = cert['processed']['cert_id'] - - just_file_name = cert['csv_scan']['link_cert_report_file_name'] - - # insert file name and identified probable certification id - if this_cert_id != "": - dot.edge(this_cert_id, this_cert_id, label=just_file_name) - - items_found_group = all_items_found[cert_long_id]['keywords_scan'] - for rules_group in items_found_group.keys(): - - # process only specified rule groups - if rules_group not in filter_rules_group: - continue - - items_found = items_found_group[rules_group] - for rule in items_found.keys(): - for match in items_found[rule]: - if match != this_cert_id: - if thick_as_occurences: - num_occurrences = str(items_found[rule][match][TAG_MATCH_COUNTER] / 3 + 1) - else: - num_occurrences = '1' - label = str(items_found[rule][match][TAG_MATCH_COUNTER]) # label with number of occurrences - if this_cert_id != "": - #if is_in_dict(cert_id_to_long_id_map, [match]): - # other_cert_node_label = get_cert_node_label(all_items_found[cert_id_to_long_id_map[match]], print_item_name) - #else: - # other_cert_node_label = match - #dot.edge(this_cert_node_label, other_cert_node_label, color='orange', style='solid', label=label, penwidth=num_occurrences) - dot.edge(this_cert_id, match, color='orange', style='solid', label=label, penwidth=num_occurrences) - - # Generate dot graph using GraphViz into pdf - dot.render(out_dot_name, view=False) - - print('{} pdf rendered'.format(out_dot_name)) - - -def plot_certid_to_item_graph(item_path, all_items_found, filter_label, out_dot_name, thick_as_occurences): - # print dot - dot = Digraph(comment='Certificate ecosystem: {}'.format(item_path)) - dot.attr('graph', label='{}'.format(filter_label), labelloc='t', fontsize='30') - dot.attr('node', style='filled') - - # insert nodes believed to be cert id for the processed certificates - for cert_long_id in all_items_found.keys(): - if is_in_dict(all_items_found[cert_long_id], ['processed', 'cert_id']): - dot.attr('node', color='green') # URL='https://www.commoncriteriaportal.org/' doesn't work for pdf - dot.node(all_items_found[cert_long_id]['processed']['cert_id']) - - dot.attr('node', color='gray') - for cert_long_id in all_items_found.keys(): - # do not continue if no values with specified path were extracted - if item_path[0] not in all_items_found[cert_long_id].keys(): - continue - - cert = all_items_found[cert_long_id] - this_cert_id = '' - if is_in_dict(cert, ['processed', 'cert_id']): - this_cert_id = cert['processed']['cert_id'] - - if is_in_dict(cert, [item_path[0], item_path[1]]): - items_found = cert[item_path[0]][item_path[1]] - for rule in items_found: - for match in items_found[rule]: - if match != this_cert_id: - if thick_as_occurences: - num_occurrences = str(items_found[rule][match][TAG_MATCH_COUNTER] / 3 + 1) - else: - num_occurrences = '1' - label = str(items_found[rule][match][TAG_MATCH_COUNTER]) # label with number of occurrences - if this_cert_id != "": - dot.edge(this_cert_id, match, color='orange', style='solid', label=label, penwidth=num_occurrences) - - # Generate dot graph using GraphViz into pdf - dot.render(out_dot_name, view=False) - print('{} pdf rendered'.format(out_dot_name)) - - -def analyze_references_graph(filter_rules_group, all_items_found, filter_label): - # build cert_id to item name mapping - certid_info = {} - for cert_long_id in all_items_found.keys(): - cert = all_items_found[cert_long_id] - if is_in_dict(cert, ['processed', 'cert_id']): - if is_in_dict(cert, ['frontpage_scan', 'cert_item']): - this_cert_id = cert['processed']['cert_id'] - if this_cert_id not in certid_info.keys(): - - certid_info[this_cert_id] = {} - certid_info[this_cert_id]['cert_item'] = cert['frontpage_scan']['cert_item'] - - # build cert_id to cert_long_id mapping - cert_id_to_long_id_mapping = {} - for cert_long_id in all_items_found.keys(): - cert = all_items_found[cert_long_id] - if is_in_dict(cert, ['processed', 'cert_id']): - if is_in_dict(cert, ['frontpage_scan', 'cert_item']): - this_cert_id = cert['processed']['cert_id'] - if this_cert_id in cert_id_to_long_id_mapping.keys(): - print('ERROR: duplicate cert_id for multiple cert_long_id: {}, {} already used by {}'.format(this_cert_id, cert_long_id, cert_id_to_long_id_mapping[this_cert_id])) - else: - cert_id_to_long_id_mapping[this_cert_id] = cert_long_id - - # build list of references - referenced_by = {} - for cert_long_id in all_items_found.keys(): - # do not continue if no keywords were extracted () - if 'keywords_scan' not in all_items_found[cert_long_id].keys(): - continue - - cert = all_items_found[cert_long_id] - this_cert_id = '' - if is_in_dict(cert, ['processed', 'cert_id']): - this_cert_id = cert['processed']['cert_id'] - - items_found_group = all_items_found[cert_long_id]['keywords_scan'] - for rules_group in items_found_group.keys(): - - # process only specified rule groups - if rules_group not in filter_rules_group: - continue - - items_found = items_found_group[rules_group] - for rule in items_found.keys(): - for match in items_found[rule]: - if match != this_cert_id: - if this_cert_id != "": - # add this_cert_id to the list of references of match item - if match not in referenced_by: - referenced_by[match] = [] - if this_cert_id not in referenced_by[match]: - referenced_by[match].append(this_cert_id) - - # - # process direct references - # - referenced_by_direct_nums = {} - for cert_id in referenced_by.keys(): - referenced_by_direct_nums[cert_id] = len(referenced_by[cert_id]) - if cert_id in cert_id_to_long_id_mapping.keys(): - all_items_found[cert_id_to_long_id_mapping[cert_id]]['processed']['direct_refs'] = len(referenced_by[cert_id]) - - print('### Certificates sorted by number of other certificates directly referencing them:') - sorted_ref_direct = sorted(referenced_by_direct_nums.items(), key=operator.itemgetter(1), reverse=False) - direct_refs = [] - for cert_id in sorted_ref_direct: - direct_refs.append(cert_id[1]) - try: - if is_in_dict(certid_info, [cert_id[0], 'cert_item']): - print(' {} : {}x directly: {}'.format(cert_id[0], cert_id[1], certid_info[cert_id[0]]['cert_item'])) - else: - print(' {} : {}x directly'.format(cert_id[0], cert_id[1])) - except UnicodeEncodeError: - print('ERROR: UnicodeEncodeError occured') - print(' Total number of certificates referenced at least once: {}'.format(len(sorted_ref_direct))) - - step = 5 - if len(direct_refs) == 0: - max_refs = 0 - else: - max_refs = max(direct_refs) + step - bins = [1, 2, 3, 4, 5] + list(range(6, max_refs + 1, step)) - compute_and_plot_hist(direct_refs, bins, 'Number of certificates', fig_label('# certificates with specific number of direct references', filter_label), 'cert_direct_refs_frequency.png') - - sanity.check_certs_referenced_once(len(sorted_ref_direct)) - - - # - # compute indirect num of references - # - referenced_by_indirect = {} - for cert_id in referenced_by.keys(): - referenced_by_indirect[cert_id] = set() - for item in referenced_by[cert_id]: - referenced_by_indirect[cert_id].add(item) - - new_change_detected = True - while new_change_detected: - new_change_detected = False - - certids_list = referenced_by.keys() - for cert_id in certids_list: - tmp_referenced_by_indirect_nums = referenced_by_indirect[cert_id].copy() - for referencing in tmp_referenced_by_indirect_nums: - if referencing in referenced_by.keys(): - tmp_referencing = referenced_by_indirect[referencing].copy() - for in_referencing in tmp_referencing: - if in_referencing not in referenced_by_indirect[cert_id]: - new_change_detected = True - referenced_by_indirect[cert_id].add(in_referencing) - - print('### Certificates sorted by number of other certificates indirectly referencing them:') - referenced_by_indirect_nums = {} - for cert_id in referenced_by_indirect.keys(): - referenced_by_indirect_nums[cert_id] = len(referenced_by_indirect[cert_id]) - if cert_id in cert_id_to_long_id_mapping.keys(): - all_items_found[cert_id_to_long_id_mapping[cert_id]]['processed']['indirect_refs'] = referenced_by_indirect_nums[cert_id] - - sorted_ref_indirect = sorted(referenced_by_indirect_nums.items(), key=operator.itemgetter(1), reverse=False) - indirect_refs = [] - for cert_id in sorted_ref_indirect: - indirect_refs.append(cert_id[1]) - try: - if is_in_dict(certid_info, [cert_id[0], 'cert_item']): - print(' {} : {}x indirectly: {}'.format(cert_id[0], cert_id[1], certid_info[cert_id[0]]['cert_item'])) - else: - print(' {} : {}x indirectly'.format(cert_id[0], cert_id[1])) - except UnicodeEncodeError: - print('ERROR: UnicodeEncodeError occured') - - step = 5 - if len(indirect_refs) == 0: - max_refs = 0 - else: - max_refs = max(indirect_refs) + step - bins = [1, 2, 3, 4, 5] + list(range(6, max_refs + 1, step)) - compute_and_plot_hist(indirect_refs, bins, 'Number of certificates', fig_label('# certificates with specific number of indirect references', filter_label), 'cert_indirect_refs_frequency.png') - - -def plot_schemes_multi_line_graph(x_ticks, data, prominent_data, x_label, y_label, title, file_name): - - figure(num=None, figsize=(16, 8), dpi=200, facecolor='w', edgecolor='k') - - line_types = ['-', ':', '-.', '--'] - num_lines_plotted = 0 - data_sorted = sorted(data.keys()) - for group in data_sorted: - items_in_year = [] - for item in sorted(data[group]): - num = len(data[group][item]) - items_in_year.append(num) - - if group in prominent_data: - plt.plot(x_ticks, items_in_year, line_types[num_lines_plotted % len(line_types)], label=group, linewidth=3) - else: - # plot minor suppliers dashed - plt.plot(x_ticks, items_in_year, line_types[num_lines_plotted % len(line_types)], label=group, linewidth=2) - - # change line type to prevent color repetitions - num_lines_plotted += 1 - - plt.rcParams.update({'font.size': 16}) - plt.legend(loc=2) - plt.xticks(x_ticks, rotation=45) - plt.xlabel(x_label) - plt.ylabel(y_label) - plt.title(title) - plt.savefig(file_name + '.png', bbox_inches='tight') - plt.savefig(file_name + '.pdf', bbox_inches='tight') - plt.close() - - -def analyze_cert_years_frequency(all_cert_items, filter_label): - scheme_date = {} - level_date = {} - category_date = {} - pp_date = {} - labs_date = {} - archive_date = {} - validity_length = {} - valid_in_years = {} - manufacturer_date = {} - manufacturer_items = {} - START_YEAR = 1997 - END_YEAR = datetime.datetime.now().year + 1 - ARCHIVE_OFFSET = 10 - - for i in range(END_YEAR - START_YEAR + ARCHIVE_OFFSET): - validity_length[i] = [] - - valid_in_years['active'] = {} - valid_in_years['archived'] = {} - for year in range(START_YEAR, END_YEAR + ARCHIVE_OFFSET): - valid_in_years['active'][year] = [] - valid_in_years['archived'][year] = [] - - for cert_long_id in all_cert_items.keys(): - cert = all_cert_items[cert_long_id] - if is_in_dict(cert, ['csv_scan', 'cc_certification_date']): - # extract year of certification - cert_date = cert['csv_scan']['cc_certification_date'] - parsed_date = parser.parse(cert_date) - cert_year = parsed_date.year - # try to extract year of archivation (if provided) - archived_year = None - if is_in_dict(cert, ['csv_scan', 'cc_archived_date']): - cert_archive_date = cert['csv_scan']['cc_archived_date'] - if cert_archive_date != '': - archived_year = parser.parse(cert_archive_date).year - - # extract EAL level - if is_in_dict(cert, ['processed', 'cc_security_level']): - level_out = cert['processed']['cc_security_level'] - - if level_out not in level_date.keys(): - level_date[level_out] = {} - for year in range(START_YEAR, END_YEAR): - level_date[level_out][year] = [] - level_date[level_out][cert_year].append(cert_long_id) - - # extract certificate category - if is_in_dict(cert, ['csv_scan', 'cc_category']): - category = cert['csv_scan']['cc_category'] - - if category not in category_date.keys(): - category_date[category] = {} - for year in range(START_YEAR, END_YEAR): - category_date[category][year] = [] - category_date[category][cert_year].append(cert_long_id) - - # extract conformance to protection profile - if is_in_dict(cert, ['csv_scan', 'cc_protection_profiles']): - pp_id = cert['csv_scan']['cc_protection_profiles'] - if pp_id == '': - pp_id = 'No Protection Profile' - else: - pp_id = 'Protection Profile' - if pp_id not in pp_date.keys(): - pp_date[pp_id] = {} - for year in range(START_YEAR, END_YEAR): - pp_date[pp_id][year] = [] - pp_date[pp_id][cert_year].append(cert_long_id) - - - # extract scheme - if is_in_dict(cert, ['csv_scan', 'cc_scheme']): - cc_scheme = cert['csv_scan']['cc_scheme'] - if cc_scheme not in scheme_date.keys(): - scheme_date[cc_scheme] = {} - for year in range(START_YEAR, END_YEAR): - scheme_date[cc_scheme][year] = [] - scheme_date[cc_scheme][cert_year].append(cert_long_id) - - # extract manufacturer(s) - if 'cc_manufacturer_simple_list' in cert['processed']: - for manufacturer in cert['processed']['cc_manufacturer_simple_list']: - if manufacturer not in manufacturer_date.keys(): - manufacturer_date[manufacturer] = {} - for year in range(START_YEAR, END_YEAR): - manufacturer_date[manufacturer][year] = [] - if manufacturer not in manufacturer_items: - manufacturer_items[manufacturer] = 0 - - manufacturer_date[manufacturer][cert_year].append(cert_long_id) - manufacturer_items[manufacturer] += 1 - - # extract laboratory - if is_in_dict(cert, ['processed', 'cert_lab']): - lab = cert['processed']['cert_lab'] - - if lab not in labs_date.keys(): - labs_date[lab] = {} - for year in range(START_YEAR, END_YEAR): - labs_date[lab][year] = [] - labs_date[lab][cert_year].append(cert_long_id) - - # extract cert archival status - if archived_year is not None: - valid_years = archived_year - cert_year + 1 - validity_length[valid_years].append(cert_long_id) - cert['processed']['cert_lifetime_length'] = valid_years - - if 'archived_date' not in archive_date.keys(): - archive_date['archived_date'] = {} - for year in range(START_YEAR, END_YEAR + ARCHIVE_OFFSET): # archive year can be quite in future - archive_date['archived_date'][year] = [] - - archive_date['archived_date'][archived_year].append(cert_long_id) - - # establish certificates active / archived in give year - for year in range(START_YEAR, END_YEAR + ARCHIVE_OFFSET): - if archived_year is not None: - # archived date is set - if year >= cert_year: - if year <= archived_year: - # certificate is valid in year - valid_in_years['active'][year].append(cert_long_id) - else: - # certificate is NOT valid in given year - valid_in_years['archived'][year].append(cert_long_id) - else: - # no archival date set => active - if year >= cert_year: - # certificate is valid in year - valid_in_years['active'][year].append(cert_long_id) - - # print manufacturers frequency - sorted_by_occurence = sorted(manufacturer_items.items(), key=operator.itemgetter(1)) - print('\n### Frequency of certificates per company') - print(' # companies: {}'.format(len(manufacturer_items))) - print(' # companies with more than 1 cert: {}'.format(len([i for i in sorted_by_occurence if i[1] > 1]))) - print(' # companies with more than 10 cert: {}'.format(len([i for i in sorted_by_occurence if i[1] > 10]))) - print(' # companies with more than 50 cert: {}\n'.format(len([i for i in sorted_by_occurence if i[1] > 50]))) - for manufacturer in sorted_by_occurence: - print(' {}: {}x'.format(manufacturer[0], manufacturer[1])) - - # plot graphs showing cert. scheme and EAL in years - years = np.arange(START_YEAR, END_YEAR) - years_extended = np.arange(START_YEAR, END_YEAR + ARCHIVE_OFFSET) - plot_schemes_multi_line_graph(years, scheme_date, ['DE', 'JP', 'FR', 'US', 'CA'], 'Year of issuance', 'Number of certificates issued', fig_label('CC certificates issuance frequency per scheme and year', filter_label), 'num_certs_in_years') - plot_schemes_multi_line_graph(years, level_date, ['EAL4+', 'EAL5+','EAL2+', 'Protection Profile'], 'Year of issuance', 'Number of certificates issued', fig_label('Certificates issuance frequency per EAL and year', filter_label), 'num_certs_eal_in_years') - plot_schemes_multi_line_graph(years, category_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Category of certificates issued in given year', filter_label), 'num_certs_category_in_years') - plot_schemes_multi_line_graph(years, pp_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Certificates with/without conforming to Protection Profile', filter_label), 'num_certs_pp_in_years') - plot_schemes_multi_line_graph(years, labs_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Number of certificates certified by laboratory in given year', filter_label), 'num_certs_by_lab_in_years') - plot_schemes_multi_line_graph(years_extended, archive_date, [], 'Year of issuance', 'Number of certificates', fig_label('Number of certificates archived or planned for archival in a given year', filter_label), 'num_certs_archived_in_years') - plot_schemes_multi_line_graph(years_extended, valid_in_years, [], 'Year', 'Number of certificates', fig_label('Number of certificates active and archived in given year', filter_label), 'num_certs_active_archived_in_years') - - sc_manufacturers = ['Gemalto', 'NXP Semiconductors', 'Samsung', 'STMicroelectronics', 'Oberthur Technologies', - 'Infineon Technologies AG', 'G+D Mobile Security GmbH', 'ATMEL Smart Card ICs', 'Idemia', - 'Athena Smartcard', 'Renesas', 'Philips Semiconductors GmbH', 'Oberthur Card Systems'] - - # plot only top manufacturers - top_manufacturers = dict(sorted_by_occurence[len(sorted_by_occurence) - 20:]).keys() # top 20 manufacturers - plot_manufacturers_date = {} - for manuf in manufacturer_date.keys(): - if manuf in top_manufacturers: - plot_manufacturers_date[manuf] = manufacturer_date[manuf] - plot_schemes_multi_line_graph(years, plot_manufacturers_date, sc_manufacturers, 'Year of issuance', 'Number of certificates issued', fig_label('Top 20 manufacturers of certified items per year', filter_label), 'manufacturer_in_years') - - # plot only smartcard manufacturers - plot_manufacturers_date = {} - for manuf in manufacturer_date.keys(): - if manuf in sc_manufacturers: - plot_manufacturers_date[manuf] = manufacturer_date[manuf] - # plot_schemes_multi_line_graph(years, plot_manufacturers_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Smartcard-related manufacturers of certified items per year', filter_label), 'manufacturer_sc_in_years') - - # plot certificate validity lengths - print('### Certificates validity period lengths:') - validity_length_numbers = [] - for length in sorted(validity_length.keys()): - print(' {} year(s): {}x {}'.format(length, len(validity_length[length]), validity_length[length])) - validity_length_numbers.append(len(validity_length[length])) - plot_bar_graph(validity_length_numbers, sorted(validity_length.keys()), 'Number of certificates', fig_label('Number of certificates with specific validity length', filter_label), 'cert_validity_length_frequency') - - -def analyze_eal_frequency(all_cert_items, filter_label): - scheme_level = {} - for cert_long_id in all_cert_items.keys(): - cert = all_cert_items[cert_long_id] - if is_in_dict(cert, ['csv_scan', 'cc_scheme']): - if is_in_dict(cert, ['processed', 'cc_security_level']): - cc_scheme = cert['csv_scan']['cc_scheme'] - level = cert['processed']['cc_security_level'] - if level.find(',') != -1: - level = level[:level.find(',')] # trim list of augmented items - if cc_scheme not in scheme_level.keys(): - scheme_level[cc_scheme] = {} - if level not in scheme_level[cc_scheme]: - scheme_level[cc_scheme][level] = 0 - scheme_level[cc_scheme][level] += 1 - - print('\n### CC EAL levels based on the certification scheme:') - for cc_scheme in sorted(scheme_level.keys()): - print(cc_scheme) - for level in sorted(scheme_level[cc_scheme].keys()): - print(' {:5}: {}x'.format(level, scheme_level[cc_scheme][level])) - - print('\n') - eal_headers = ['EAL0+', 'EAL1', 'EAL1+','EAL2', 'EAL2+','EAL3', 'EAL3+','EAL4', 'EAL4+','EAL5', - 'EAL5+','EAL6', 'EAL6+','EAL7', 'EAL7+', 'None'] - - total_eals = {} - for level in eal_headers: - total_eals[level] = 0 - - cc_eal_freq = [] - sum_total = 0 - for cc_scheme in sorted(scheme_level.keys()): - this_scheme_levels = [cc_scheme] - total = 0 - for level in eal_headers: - if level in scheme_level[cc_scheme]: - this_scheme_levels.append(scheme_level[cc_scheme][level]) - total += scheme_level[cc_scheme][level] - total_eals[level] += scheme_level[cc_scheme][level] - else: - this_scheme_levels.append(0) - - this_scheme_levels.append(total) - sum_total += total - cc_eal_freq.append(this_scheme_levels) - - total_eals_row = [] - for level in sorted(total_eals.keys()): - total_eals_row.append(total_eals[level]) - - # plot bar graph with frequency of CC EAL levels - plot_bar_graph(total_eals_row, eal_headers, 'Number of certificates', fig_label('Number of certificates of specific EAL level', filter_label), 'cert_eal_frequency') - - # Print table with results over national schemes - total_eals_row.append(sum_total) - cc_eal_freq.append(['Total'] + total_eals_row) - print(tabulate(cc_eal_freq, ['CC scheme'] + eal_headers + ['Total'])) - - -def analyze_pdfmeta(all_cert_items, filter_label): - pdf_tags = {} - pdf_values = {} - for cert_long_id in all_cert_items.keys(): - cert = all_cert_items[cert_long_id] - - if is_in_dict(cert, ['pdfmeta_scan']): - pdf_scan = cert['pdfmeta_scan'] - for tag in pdf_scan.keys(): - if tag not in pdf_tags.keys(): - pdf_tags[tag] = 1 # count number of occurence of particular pdf tag - pdf_values[tag] = {} # collect values for particular tag and their frequencies - pdf_values[tag][pdf_scan[tag]] = [cert_long_id] - else: - pdf_tags[tag] += 1 - if pdf_scan[tag] not in pdf_values[tag].keys(): - pdf_values[tag][pdf_scan[tag]] = [cert_long_id] - else: - pdf_values[tag][pdf_scan[tag]].append(cert_long_id) - - - print('\n### Extracted pdf tags frequency:') - sorted_by_occurence = sorted(pdf_tags.items(), key=operator.itemgetter(1)) - for tag in sorted_by_occurence: - print('{:10}: {}x'.format(tag[0], tag[1])) - if tag[0] in ['pdf_file_size_bytes', '/ModDate', 'pdf_number_of_pages', '/CreationDate']: - print(' Would be too many separate outputs, print supressed') - else: - detected_items_count = {} - for item in pdf_values[tag[0]].items(): - detected_items_count[item[0]] = len(item[1]) - sorted_by_occurence_values = sorted(detected_items_count.items(), key=operator.itemgetter(1)) - for value in sorted_by_occurence_values: - print(' {:10}: {}x : {}'.format(value[0], value[1], pdf_values[tag[0]][value[0]])) - - -def analyze_security_assurance_component_frequency(all_cert_items, filter_label): - return analyze_sc_frequency(all_cert_items, filter_label, 'assurance') - - -def analyze_security_functional_component_frequency(all_cert_items, filter_label): - return analyze_sc_frequency(all_cert_items, filter_label, 'functional') - - -def analyze_sc_frequency(all_cert_items, filter_label, sec_component_label): - sars_freq = {} - key_name = 'invalid' - shortcut = 'invalid' - if sec_component_label == 'functional': - key_name = 'rules_security_functional_components' - shortcut = 'sfr' - if sec_component_label == 'assurance': - key_name = 'rules_security_assurance_components' - shortcut = 'sar' - - for cert_long_id in all_cert_items.keys(): - cert = all_cert_items[cert_long_id] - if is_in_dict(cert, ['keywords_scan', key_name]): - sars = cert['keywords_scan'][key_name] - for sar_rule in sars: - for sar_hit in sars[sar_rule]: - if sar_hit not in sars_freq.keys(): - sars_freq[sar_hit] = 0 - sars_freq[sar_hit] += 1 - - - print('\n### CC security ' + sec_component_label + ' components frequency:') - sars_labels = sorted(sars_freq.keys()) - sars_freq_nums = [] - for sar in sars_labels: - print('{:10}: {}x'.format(sar, sars_freq[sar])) - sars_freq_nums.append(sars_freq[sar]) - - print('\n### CC security ' + sec_component_label + ' components frequency sorted by num occurences:') - sorted_by_occurence = sorted(sars_freq.items(), key=operator.itemgetter(1)) - for sar in sorted_by_occurence: - print('{:10}: {}x'.format(sar[0], sar[1])) - - # plot bar graph with frequency of CC SARs - plot_bar_graph(sars_freq_nums, sars_labels, 'Number of certificates', fig_label('Number of certificates mentioning specific security ' + sec_component_label + ' component (' + shortcut + ')\nAll listed occured at least once', filter_label), 'cert_' + shortcut + '_frequency') - if len(sars_freq_nums) > 0 and len(sars_labels) > 0: - sars_freq_nums, sars_labels = (list(t) for t in zip(*sorted(zip(sars_freq_nums, sars_labels), reverse=True))) - plot_bar_graph(sars_freq_nums, sars_labels, 'Number of certificates', fig_label( - 'Number of certificates mentioning specific security ' + sec_component_label + ' component (' + shortcut + ')\nAll listed occured at least once', - filter_label), 'cert_' + shortcut + '_frequency_sorted') - else: - print('ERROR: len(sars_freq_nums) < 1') - - # plot heatmap of SARs frequencies based on type (row) and level (column) - sars_labels = sorted(sars_freq.keys()) - sars_unique_names = [] - for sar in sars_labels: - if sar.find('.') != -1: - name = sar[:sar.find('.')] - else: - name = sar - if name not in sars_unique_names: - sars_unique_names.append(name) - - sars_unique_names = sorted(sars_unique_names) - max_sar_level = 8 - num_sars = len(sars_unique_names) - sar_heatmap = [] - sar_matrix = [] - for i in range(1, max_sar_level + 1): - sar_row = [] - for name in sars_unique_names: - sar_row.append(0) - sar_matrix.append(sar_row) - - for sar in sorted_by_occurence: - if sar[0].find('.') != -1: - name = sar[0][:sar[0].find('.')] - name_index = sars_unique_names.index(name) - level = int(sar[0][sar[0].find('.') + 1:]) - sar_matrix[level - 1][name_index] = sar[1] - - # plot heatmap graph with frequency of SAR levels - y_data_labels = range(1, max_sar_level + 2) - plot_heatmap_graph(sar_matrix, sars_unique_names, y_data_labels, 'Security ' + sec_component_label + ' component (' + shortcut + ') class', 'Security ' + sec_component_label + ' components (' + shortcut + ') level', fig_label('Frequency of achieved levels for Security ' + sec_component_label + ' component (' + shortcut + ') classes', filter_label), 'cert_' + shortcut + '_heatmap') - - -def generate_dot_graphs(all_items_found, filter_label): - # with name of certified items - print_dot_graph(['rules_cert_id'], all_items_found, filter_label, 'certidname_graph.dot', True, True) - # without name of certified items - print_dot_graph(['rules_cert_id'], all_items_found, filter_label, 'certid_graph.dot', True, False) - # link between device and its javacard version - print_dot_graph(['rules_javacard'], all_items_found, filter_label, 'cert_javacard_graph.dot', False, True) - - # print_dot_graph(['rules_security_level'], all_items_found, filter_label, 'cert_security_level_graph.dot', True) - # print_dot_graph(['rules_crypto_libs'], all_items_found, filter_label, 'cert_crypto_libs_graph.dot', False) - # print_dot_graph(['rules_vendor'], all_items_found, filter_label, 'rules_vendor.dot', False) - # print_dot_graph(['rules_crypto_algs'], all_items_found, filter_label, 'rules_crypto_algs.dot', False) - # print_dot_graph(['rules_protection_profiles'], all_items_found, filter_label, 'rules_protection_profiles.dot', False) - # print_dot_graph(['rules_defenses'], all_items_found, filter_label, 'rules_defenses.dot', False) - diff --git a/src/cert_rules.py b/src/cert_rules.py deleted file mode 100644 index 8623996c..00000000 --- a/src/cert_rules.py +++ /dev/null @@ -1,357 +0,0 @@ -rules_cert_id = [ - 'BSI-DSZ-CC-[0-9]+?-[0-9]+', # German BSI - 'BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+-[0-9]+', # German BSI - 'BSI-DSZ-CC-[0-9]+?-(?:V|v)[0-9]+', # German BSI - 'BSI [0-9]+?', # German BSI - # 'CC-Zert-.+?', - 'ANSSI(?:-|-CC-)[0-9]+?/[0-9]+', # French - # 'ANSSI-CC-CER-F-.+?', # French - 'DCSSI-[0-9]+?/[0-9]+?', # French - 'Certification Report [0-9]+?/[0-9]+?', # French - 'Rapport de certification [0-9]+?/[0-9]+?', # French - 'NSCIB-CC-[0-9][0-9][0-9][0-9].+?', # Netherlands - 'NSCIB-CC-[0-9][0-9][0-9][0-9][0-9]*-CR', # Netherlands - 'NSCIB-CC-[0-9][0-9]-[0-9]+?-CR[0-9]+?', # Netherlands - 'SERTIT-[0-9]+?', # Norway - 'CCEVS-VR-(?:|VID)[0-9]+?-[0-9]+?', # US NSA - # '[0-9][0-9\-]+?-CR', # Canada - 'CRP[0-9][0-9][0-9][0-9]*?', # UK CESG - 'CERTIFICATION REPORT No. P[0-9]+?', # UK CESG - '20[0-9][0-9]-[0-9]+-INF-[0-9]+?', # Spain - 'KECS-CR-[0-9]+?-[0-9]+?', # Korea - 'KECS-ISIS-[0-9]+?-[0-9][0-9][0-9][0-9]', # Korea - 'CRP-C[0-9]+?-[0-9]+?', # Japan - 'ISCB-[0-9]+?-RPT-[0-9]+?', # Malaysia - 'OCSI/CERT/.+?', # Italia - '[0-9\\.]+?/TSE-CCCS-[0-9]+?', # Turkis CCCS - 'BTBD-.+?', # Turkis CCCS -] - -rules_vendor = [ - 'NXP', - 'Infineon', - 'Samsung', - '(?:STMicroelectronics|STM)', - 'Feitian', - 'Gemalto', - 'Gemplus', - 'Axalto', - '(?:Oberthur|OBERTHUR)', - '(?:Idemia|IDEMIA)', - '(?:G\&D|G\+D|Giesecke+Devrient|Giesecke \& Devrient)', - 'Philips', - 'Sagem', -] - -rules_eval_facilities = [ - 'Serma Technologies', - 'THALES - CEACI' -] - -rules_protection_profiles = [ - 'BSI-(?:CC[-_]|)PP[-_]*.+?', - 'PP-SSCD.+?', - 'PP_DBMS_.+?' - # 'Protection Profile', - 'CCMB-20.+?', - 'CCMB-20[0-9]+?-[0-9]+?-[0-9]+?', - 'BSI-CCPP-.+?', - 'ANSSI-CC-PP.+?', - 'WBIS_V[0-9]\\.[0-9]', - 'EHCT_V.+?' -] - -rules_technical_reports = [ - 'BSI[ ]*TR-[0-9]+?(?:-[0-9]+?|)', -] - -rules_device_id = [ - 'G87-.+?', - 'ATMEL AT.+?', -] - -rules_os = [ - 'STARCOS(?: [0-9\\.]+?|)', - 'JCOP[ ]*[0-9]' -] - -rules_standard_id = [ - 'FIPS ?(?:PUB )?[0-9]+-[0-9]+?', - 'FIPS ?(?:PUB )?[0-9]+?', - 'NIST SP [0-9]+-[0-9]+?[a-zA-Z]?', - 'PKCS[ #]*[1-9]+', - 'TLS[ ]*v[0-9\\.]+', - 'TLS[ ]*v[0-9\\.]+', - 'BSI-AIS[ ]*[0-9]+?', - 'AIS[ ]*[0-9]+?', - 'RFC[ ]*[0-9]+?', - 'ISO/IEC[ ]*[0-9]+[-]*[0-9]*', - 'ISO/IEC[ ]*[0-9]+:[ 0-9]+', - 'ISO/IEC[ ]*[0-9]+', - 'ICAO(?:-SAC|)', - '[Xx]\\.509', - 'RFC [0-9]+' -] - -rules_security_level = [ - 'EAL[ ]*[0-9+]+?', - 'EAL[ ]*[0-9] augmented+?', - 'ITSEC[ ]*E[1-9]*.+?', -] - -rules_security_assurance_components = [ - r'ACE_[A-Z]{3}(?:\.[0-9]|)', - r'ACM_[A-Z]{3}(?:\.[0-9]|)', - r'ACO_[A-Z]{3}(?:\.[0-9]|)', - r'ADO_[A-Z]{3}(?:\.[0-9]|)', - r'ADV_[A-Z]{3}(?:\.[0-9]|)', - r'AGD_[A-Z]{3}(?:\.[0-9]|)', - r'ALC_[A-Z]{3}(?:\.[0-9]|)', - r'ATE_[A-Z]{3}(?:\.[0-9]|)', - r'AVA_[A-Z]{3}(?:\.[0-9]|)', - r'AMA_[A-Z]{3}(?:\.[0-9]|)', - r'APE_[A-Z]{3}(?:\.[0-9]|)', - r'ASE_[A-Z]{3}(?:\.[0-9]|)' -] - -rules_security_functional_components = [ - r'FAU_[A-Z]{3}(?:\.[0-9]|)', - r'FCO_[A-Z]{3}(?:\.[0-9]|)', - r'FCS_[A-Z]{3}(?:\.[0-9]|)', - r'FDP_[A-Z]{3}(?:\.[0-9]|)', - r'FIA_[A-Z]{3}(?:\.[0-9]|)', - r'FMT_[A-Z]{3}(?:\.[0-9]|)', - r'FPR_[A-Z]{3}(?:\.[0-9]|)', - r'FPT_[A-Z]{3}(?:\.[0-9]|)', - r'FRU_[A-Z]{3}(?:\.[0-9]|)', - r'FTA_[A-Z]{3}(?:\.[0-9]|)', - r'FTP_[A-Z]{3}(?:\.[0-9]|)' -] - -rules_javacard = [ - #'(?:Java Card|JavaCard)', - #'(?:Global Platform|GlobalPlatform)', - r'(?:Java Card|JavaCard) [2-3]\.[0-9](?:\.[0-9]|)', - r'(?:Java Card|JavaCard) \(version [2-3]\.[0-9](?:\.[0-9]|)\)', - r'(?:Global Platform|GlobalPlatform) [2-3]\.[0-9]\.[0-9]', - r'(?:Global Platform|GlobalPlatform) \(version [2-3]\.[0-9]\.[0-9]\)', -] - -rules_crypto_algs = [ - 'RSA[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)', - 'RSASSAPKCS1-[Vv]1_5', - 'SHA[-]*(?:160|224|256|384|512)', - 'AES[-]*(?:128|192|256|)', - 'SHA-1', - 'MD5', - 'HMAC', - 'HMAC-SHA-(?:160|224|256|384|512)', - '(Diffie-Hellman|DH)', - 'ECDH', - 'ECDSA', - 'EdDSA', - '3?DES', - 'ECC', - 'DTRNG', - 'TRNG', - 'RN[GD]', - 'RBG', -] - -rules_block_cipher_modes = [ - 'ECB', - 'CBC', - 'CTR', - 'CFB', - 'OFB', - 'GCM', -] - -rules_ecc_curves = [ - 'P-(192|224|256|384|521)', - 'brainpoolP[0-9]{3}[rkt][12]', - '(sec|ansi)[pt].+?[rk][12]', - 'prime[0-9]{3}v[123]', - 'c2[pto]nb[0-9]{3}[vw][123]', - 'FRP256v1', - 'Curve(25519|1174|4417|22103|67254|383187|41417)', - 'Ed(25519|448)', - 'ssc-(160|192|224|256|288|320|384|512)', - 'Tweedle(dee|dum)', - 'JubJub', - 'BLS(12|24)-[0-9]{3}' -] - -rules_cplc = [ - 'IC[ ]*Fabricator', - 'IC[ ]*Type', - 'IC[ ]*Version', -] - -rules_crypto_engines = [ - 'TORNADO', - 'SmartMX', - 'SmartMX2' - 'NesCrypt', -] - -rules_crypto_libs = [ - '(?:NesLib|NESLIB) [v]*[0-9.]+', - 'AT1 Secure .{1,30}? Library [v]*[0-9.]+', - 'AT1 Secure RSA/ECC/SHA library', - 'Crypto Library [v]*[0-9.]+', - 'ATMEL Toolbox [0-9.]+', - 'v1.02.013' # Infineon's ROCA-vulnerable library -] - -rules_IC_data_groups = [ - r'EF\.DG[1-9][0-6]?', - r'EF\.COM', - r'EF\.CardAccess', - r'EF\.SOD', - r'EF\.ChipSecurity' -] - -rules_defenses = [ - '[Mm]alfunction', - 'Leak-Inherent', - '[Pp]hysical [Pp]robing', - '[pP]hysical [tT]ampering', - '[Ss]ide.channels?', - 'SPA', - 'DPA', - 'DFA', - '[Ff]+ault induction', - 'ROCA', -] - -rules_certification_process = [ - '[oO]ut of [sS]cope', - '[\\.\\(].{0,100}?.[oO]ut of [sS]cope..{0,100}?[\\.\\)]', - '.{0,100}[oO]ut of [sS]cope.{0,100}', - '.{0,100}confidential document.{0,100}', - '[sS]ecurity [fF]unction SF\\.[a-zA-Z0-9_]', -] - -rules_vulnerabilities = [ - 'CVE-[0-9]+?-[0-9]+?', - 'CWE-[0-9]+?', -] - -rules_other = [ - 'library', - # 'http[s]*://.+?/' -] - -rules_fips_remove_algorithm_ids = [ - r"HMAC(?:-SHA)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"HMAC(?:-SHA)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"HMAC(?:-SHA)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"HMAC(?:-SHA)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", - r"SH[SA][-]*(?:160|224|256|384|512)?(?:[\s]*?(?:KAT)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)(\d{4})(?:\)?\[#?\d+\])?", - r"SH[SA][-]*(?:160|224|256|384|512)?(?:[\s]*?(?:KAT)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)(\d{3})(?:\)?\[#?\d+\])?", - r"SH[SA][-]*(?:160|224|256|384|512)?(?:[\s]*?(?:KAT)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)(\d{2})(?:\)?\[#?\d+\])?", - r"SH[SA][-]*(?:160|224|256|384|512)?(?:[\s]*?(?:KAT)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)(\d{1})(?:\)?\[#?\d+\])?", - r"RSA(?:[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"RSA(?:[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"RSA(?:[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"RSA(?:[- ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", - r"(?:RSA)?[- ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{4})?", - r"(?:RSA)?[- ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{3})?", - r"(?:RSA)?[- ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{2})?", - r"(?:RSA)?[- ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{1})?", - r"AES[- ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR)*?[,\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})(?:\)?\[#?\d+\])?", - r"AES[- ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR)*?[,\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})(?:\)?\[#?\d+\])?", - r"AES[- ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR)*?[,\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})(?:\)?\[#?\d+\])?", - r"AES[- ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR)*?[,\s(]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})(?:\)?\[#?\d+\])?", - r"Diffie[- ]*Hellman[,\s(]*?(?:CVL|\s)*?(?:#|Cert\.?|Certificate)?[\s#]*?\s?(\d{4})", - r"Diffie[- ]*Hellman[,\s(]*?(?:CVL|\s)*?(?:#|Cert\.?|Certificate)?[\s#]*?\s?(\d{3})", - r"Diffie[- ]*Hellman[,\s(]*?(?:CVL|\s)*?(?:#|Cert\.?|Certificate)?[\s#]*?\s?(\d{2})", - r"Diffie[- ]*Hellman[,\s(]*?(?:CVL|\s)*?(?:#|Cert\.?|Certificate)?[\s#]*?\s?(\d{1})", - r"DRBG[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"DRBG[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"DRBG[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"DRBG[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", - r"DES[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"DES[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"DES[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"DES[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s]*?(?:#|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", - r"CVL[\s#]*?(\d{4})", - r"CVL[\s#]*?(\d{3})", - r"CVL[\s#]*?(\d{2})", - r"CVL[\s#]*?(\d{1})", - r"PAA[: #]*?\d{4}", - r"PAA[: #]*?\d{3}", - r"PAA[: #]*?\d{2}", - r"PAA[: #]*?\d{1}", - r"(?:#|Cert\.?|Certificate)[\s#]*?(\d+)?\s*?(?:AES|SHS|SHA|RSA|HMAC|Diffie-Hellman|DRBG|DES|CVL)", - r"PKCS[ ]?#?\d+", - r"Survey #192" # why would they get an address like this /o\ cert 2079 -] -rules_fips_cert = [ - # r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P\d{4})", - # r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P\d{3})", - # r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P\d{2})", - # r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P\d{1}) - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P\d{4})", - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P\d{3})", - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P\d{2})", - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P\d{1})" -] - -# rule still too "general" -rules_fips_security_level = [ - r"[lL]evel (\d)" -] - -rules_fips_htmls = [ - r"module-name\">\s*(?P[^<]*)", - r"module-standard\">\s*(?P[^<]*)", - r"Status[\s\S]*?\">\s*(?P[^<]*)", - r"Sunset Date[\s\S]*?\">\s*(?P[^<]*)", - r"Validation Dates[\s\S]*?\">\s*(?P[^<]*)", - r"Overall Level[\s\S]*?\">\s*(?P[^<]*)", - r"Caveat[\s\S]*?\">\s*(?P[^<]*)", - r"Security Level Exceptions[\s\S]*?\">\s*(?P)", - r"Module Type[\s\S]*?\">\s*(?P[^<]*)", - r"Embodiment[\s\S]*?\">\s*(?P[^<]*)", - r"Tested Configuration[\s\S]*?\">\s*(?P)", - r"FIPS Algorithms[\s\S]*?\">\s*(?P[\s\S]*)", - r"Allowed Algorithms[\s\S]*?\">\s*(?P[^<]*)", - r"Software Versions[\s\S]*?\">\s*(?P[^<]*)", - r"Product URL[\s\S]*?\">\s*.*)\"", - r"Vendor<\/h4>[\s\S]*?href=\".*?\">(?P.*?)<\/a>" -] - -# rules_security_target_class -rules = {} -rules['rules_vendor'] = rules_vendor -rules['rules_cert_id'] = rules_cert_id -rules['rules_protection_profiles'] = rules_protection_profiles -rules['rules_technical_reports'] = rules_technical_reports -rules['rules_device_id'] = rules_device_id -rules['rules_os'] = rules_os -rules['rules_standard_id'] = rules_standard_id -rules['rules_security_level'] = rules_security_level -rules['rules_security_assurance_components'] = rules_security_assurance_components -rules['rules_security_functional_components'] = rules_security_functional_components -rules['rules_javacard'] = rules_javacard -rules['rules_crypto_algs'] = rules_crypto_algs -rules['block_cipher_modes'] = rules_block_cipher_modes -rules['rules_ecc_curves'] = rules_ecc_curves -rules['rules_cplc'] = rules_cplc -rules['rules_crypto_engines'] = rules_crypto_engines -rules['rules_crypto_libs'] = rules_crypto_libs -rules['rules_IC_data_groups'] = rules_IC_data_groups -rules['rules_defenses'] = rules_defenses -rules['rules_certification_process'] = rules_certification_process -rules['rules_vulnerabilities'] = rules_vulnerabilities -rules['rules_other'] = rules_other - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# For FIPS -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -fips_rules = {} -fips_rules['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids -fips_rules['rules_security_level'] = rules_fips_security_level -fips_rules['rules_cert_id'] = rules_fips_cert diff --git a/src/download_files.py b/src/download_files.py deleted file mode 100644 index a877593c..00000000 --- a/src/download_files.py +++ /dev/null @@ -1,36 +0,0 @@ -import extract_certificates - -FILE_ERRORS_STRATEGY = extract_certificates.FILE_ERRORS_STRATEGY - - -def generate_fips_basic_download_script(): - with open('download_fips_web.bat', 'w', errors=FILE_ERRORS_STRATEGY) as file: - file.write( - 'curl "https://csrc.nist.gov/projects/cryptographic-module-validation-program/validated-modules/search' - '/all" -o fips_modules_validated.html\n') - - -def generate_fips_download_script(file_name, fips_dir): - """generate_fips_download_script. - - :param file_name: name of the download file - :param fips_dir: directory for saved files - """ - html_dir = fips_dir + '/html/' - sp_dir = fips_dir + '/security_policies/' - - with open(file_name, 'w', errors=FILE_ERRORS_STRATEGY) as write_file: - # make directories for both html and security policies, scraping in one go - write_file.write('mkdir {}\n'.format(html_dir)) - write_file.write('mkdir {}\n\n'.format(sp_dir)) - - for cert_id in range(1, 4001): - write_file.write( - 'curl "https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/{}" -o {}{}.html\n'.format( - cert_id, html_dir, cert_id)) - write_file.write( - 'curl "https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents' - '/security-policies/140sp{}.pdf" -o {}{}.pdf\n'.format( - cert_id, sp_dir, cert_id)) - write_file.write("{} {}{}.pdf\n".format( - extract_certificates.PDF2TEXT_CONVERT, sp_dir, cert_id)) diff --git a/src/extract_certificates.py b/src/extract_certificates.py deleted file mode 100644 index 07ff1707..00000000 --- a/src/extract_certificates.py +++ /dev/null @@ -1,2435 +0,0 @@ -from PyPDF2 import PdfFileReader - -import sanity -from tags_constants import * -import re -import os -import operator -from graphviz import Digraph -import json -import csv -import string - -from analyze_certificates import is_in_dict -from cert_rules import rules, fips_rules -from enum import Enum -import matplotlib.pyplot as plt - -plt.rcdefaults() - -# if True, then exception is raised when unexpect intermediate number is obtained -# Used as sanity check during development to detect sudden drop in number of extracted features -APPEND_DETAILED_MATCH_MATCHES = False -VERBOSE = False -FILE_ERRORS_STRATEGY = 'surrogateescape' -'replace' -# FILE_ERRORS_STRATEGY = 'strict' -CC_WEB_URL = 'https://www.commoncriteriaportal.org' -PDF2TEXT_CONVERT = 'pdftotext -raw' - -REGEXEC_SEP = '[ ,;\]”)(]' -LINE_SEPARATOR = ' ' -# LINE_SEPARATOR = '' # if newline is not replaced with space, long string included in matches are found - -printable = set(string.printable) - - -def search_files(folder): - for root, dirs, files in os.walk(folder): - yield from [os.path.join(root, x) for x in files] - - -def get_line_number(lines, line_length_compensation, match_start_index): - line_chars_offset = 0 - line_number = 1 - for line in lines: - line_chars_offset += len(line) + line_length_compensation - if line_chars_offset > match_start_index: - # we found the line - return line_number - line_number += 1 - # not found - return -1 - - -def load_cert_file(file_name, limit_max_lines=-1, line_separator=LINE_SEPARATOR): - lines = [] - was_unicode_decode_error = False - with open(file_name, 'r', errors=FILE_ERRORS_STRATEGY) as f: - try: - lines = f.readlines() - except UnicodeDecodeError: - f.close() - was_unicode_decode_error = True - print(' WARNING: UnicodeDecodeError, opening as utf8') - - with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2: - # coding failure, try line by line - line = ' ' - while line: - try: - line = f2.readline() - lines.append(line) - except UnicodeDecodeError: - # ignore error - continue - - whole_text = '' - whole_text_with_newlines = '' - # we will estimate the line for searched matches - # => we need to known how much lines were modified (removal of eoln..) - # for removed newline and for any added separator - line_length_compensation = 1 - len(LINE_SEPARATOR) - lines_included = 0 - for line in lines: - if limit_max_lines != -1 and lines_included >= limit_max_lines: - break - - whole_text_with_newlines += line - line = line.replace('\n', '') - whole_text += line - whole_text += line_separator - lines_included += 1 - - return whole_text, whole_text_with_newlines, was_unicode_decode_error - - -def load_cert_html_file(file_name): - with open(file_name, 'r', errors=FILE_ERRORS_STRATEGY) as f: - try: - whole_text = f.read() - except UnicodeDecodeError: - f.close() - with open(file_name, encoding="utf8", errors=FILE_ERRORS_STRATEGY) as f2: - try: - whole_text = f2.read() - except UnicodeDecodeError: - print('### ERROR: failed to read file {}'.format(file_name)) - return whole_text - - -def normalize_match_string(match): - # normalize match - match = match.strip() - match = match.rstrip(']') - match = match.rstrip(os.sep) - match = match.rstrip(';') - match = match.rstrip('.') - match = match.rstrip('”') - match = match.rstrip('"') - match = match.rstrip(':') - match = match.rstrip(')') - match = match.rstrip('(') - match = match.rstrip(',') - match = match.replace(' ', ' ') # two spaces into one - - sanitized = ''.join(filter(lambda x: x in printable, match)) - - return sanitized - - -def set_match_string(items, key_name, new_value): - if key_name not in items.keys(): - items[key_name] = new_value - else: - old_value = items[key_name] - if old_value != new_value: - print(' WARNING: values mismatch, key=\'{}\', old=\'{}\', new=\'{}\''.format( - key_name, old_value, new_value)) - - -def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR, - should_censure_right_away=False, fips_items=None, ): - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file( - file_name, limit_max_lines, line_separator) - - file_name = os.path.splitext(os.path.splitext(os.path.basename(file_name))[0])[0] - # apply all rules - items_found_all = {} - for rule_group in search_rules.keys(): - if rule_group not in items_found_all: - items_found_all[rule_group] = {} - - items_found = items_found_all[rule_group] - - for rule in search_rules[rule_group]: - - rule_and_sep = rule + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text_with_newlines): - # insert rule if at least one match for it was found - if rule not in items_found: - items_found[rule] = {} - - match = m.group() - match = normalize_match_string(match) - is_algorithm = False - if fips_items and match != '': - certs = [x['Certificate'] - for x in fips_items[file_name]['fips_algorithms']] - - match_cert_id = ''.join(filter(str.isdigit, match)) - # if file_name == '/home/stan/sec-certs-master/files/fips/security_policies/3676.html.txt': - - for fips_cert in certs: - for actual_cert in fips_cert: - if actual_cert != '' and match_cert_id == ''.join(filter(str.isdigit, actual_cert)): - is_algorithm = True - - if is_algorithm: - continue - - if match not in items_found[rule]: - items_found[rule][match] = {} - items_found[rule][match][TAG_MATCH_COUNTER] = 0 - if APPEND_DETAILED_MATCH_MATCHES: - items_found[rule][match][TAG_MATCH_MATCHES] = [] - # else: - # items_found[rule][match][TAG_MATCH_MATCHES] = ['List of matches positions disabled. Set APPEND_DETAILED_MATCH_MATCHES to True'] - - items_found[rule][match][TAG_MATCH_COUNTER] += 1 - match_span = m.span() - # estimate line in original text file - # line_number = get_line_number(lines, line_length_compensation, match_span[0]) - # start index, end index, line number - # items_found[rule][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number]) - if APPEND_DETAILED_MATCH_MATCHES: - items_found[rule][match][TAG_MATCH_MATCHES].append( - [match_span[0], match_span[1]]) - if should_censure_right_away: - whole_text_with_newlines = whole_text_with_newlines.replace( - match, 'x' * len(match)) - - # highlight all found strings from the input text and store the rest - if not should_censure_right_away: - for rule_group in items_found_all.keys(): - items_found = items_found_all[rule_group] - for rule in items_found.keys(): - for match in items_found[rule]: - # warning - if AES string is removed before AES-128, -128 will be left in text (does it matter?) - whole_text_with_newlines = whole_text_with_newlines.replace( - match, 'x' * len(match)) - - return items_found_all, (whole_text_with_newlines, was_unicode_decode_error) - - -def print_total_matches_in_files(all_items_found_count): - sorted_all_items_found_count = sorted( - all_items_found_count.items(), key=operator.itemgetter(1)) - for file_name_count in sorted_all_items_found_count: - print('{:03d}: {}'.format(file_name_count[1], file_name_count[0])) - - -def print_total_found_cert_ids(all_items_found_certid_count): - sorted_certid_count = sorted( - all_items_found_certid_count.items(), key=operator.itemgetter(1), reverse=True) - for file_name_count in sorted_certid_count: - print('{:03d}: {}'.format(file_name_count[1], file_name_count[0])) - - -def print_guessed_cert_id(cert_id): - sorted_cert_id = sorted(cert_id.items(), key=operator.itemgetter(1)) - for double in sorted_cert_id: - just_file_name = double[0] - if just_file_name.rfind(os.sep) != -1: - just_file_name = just_file_name[just_file_name.rfind(os.sep) + 1:] - print('{:30s}: {}'.format(double[1], just_file_name)) - - -def print_all_results(items_found_all): - # print results - for rule_group in items_found_all.keys(): - print(rule_group) - items_found = items_found_all[rule_group] - for rule in items_found.keys(): - print(' ' + rule) - for match in items_found[rule]: - print(' {}: {}'.format(match, items_found[rule][match])) - - -def count_num_items_found(items_found_all): - num_items_found = 0 - for rule_group in items_found_all.keys(): - items_found = items_found_all[rule_group] - for rule in items_found.keys(): - for match in items_found[rule]: - num_items_found += 1 - - return num_items_found - - -def estimate_cert_id(frontpage_scan, keywords_scan, file_name): - # check if cert id was extracted from frontpage (most priority) - frontpage_cert_id = '' - if frontpage_scan != None: - if 'cert_id' in frontpage_scan.keys(): - frontpage_cert_id = frontpage_scan['cert_id'] - - keywords_cert_id = '' - if keywords_scan != None: - # find certificate ID which is the most common - num_items_found_certid_group = 0 - max_occurences = 0 - items_found = keywords_scan['rules_cert_id'] - for rule in items_found.keys(): - for match in items_found[rule]: - num_occurences = items_found[rule][match][TAG_MATCH_COUNTER] - if num_occurences > max_occurences: - max_occurences = num_occurences - keywords_cert_id = match - num_items_found_certid_group += num_occurences - if VERBOSE: - print(' -> most frequent cert id: {}, {}x'.format(keywords_cert_id, - num_items_found_certid_group)) - - # try to search for certificate id directly in file name - if found, higher priority - filename_cert_id = '' - if file_name != None: - file_name_no_suff = file_name[:file_name.rfind('.')] - file_name_no_suff = file_name_no_suff[file_name_no_suff.rfind( - os.sep) + 1:] - for rule in rules['rules_cert_id']: - file_name_no_suff += ' ' - matches = re.findall(rule, file_name_no_suff) - if len(matches) > 0: - # we found cert id directly in name - # print(' -> cert id found directly in certificate name: {}'.format(matches[0])) - filename_cert_id = matches[0] - - if VERBOSE: - print('Identified cert ids for {}:'.format(file_name)) - print(' frontpage_cert_id: {}'.format(frontpage_cert_id)) - print(' filename_cert_id: {}'.format(filename_cert_id)) - print(' keywords_cert_id: {}'.format(keywords_cert_id)) - - if frontpage_cert_id != '': - return frontpage_cert_id - if filename_cert_id != '': - return filename_cert_id - if keywords_cert_id != '': - return keywords_cert_id - - return '' - - -def save_modified_cert_file(target_file, modified_cert_file_text, is_unicode_text): - if is_unicode_text: - write_file = open(target_file, "w", encoding="utf8", errors="replace") - else: - write_file = open(target_file, "w", errors="replace") - - try: - write_file.write(modified_cert_file_text) - except UnicodeEncodeError as e: - write_file.close() - print('UnicodeDecodeError while writing file fragments back') - - write_file.close() - - -def process_raw_header(items_found): - return items_found - - -def print_specified_property_sorted(section_name, item_name, items_found_all): - specific_item_values = [] - for file_name in items_found_all.keys(): - if section_name in items_found_all[file_name].keys(): - if item_name in items_found_all[file_name][section_name].keys(): - specific_item_values.append( - items_found_all[file_name][item_name]) - else: - print('WARNING: Item {} not found in file {}'.format( - item_name, file_name)) - - print('*** Occurrences of *{}* item'.format(item_name)) - sorted_items = sorted(specific_item_values) - for item in sorted_items: - print(item) - - -def print_found_properties(items_found_all): - print_specified_property_sorted(TAG_CERT_ID, items_found_all) - print_specified_property_sorted(TAG_CERT_ITEM, items_found_all) - print_specified_property_sorted(TAG_CERT_ITEM_VERSION, items_found_all) - print_specified_property_sorted( - TAG_REFERENCED_PROTECTION_PROFILES, items_found_all) - print_specified_property_sorted(TAG_CC_VERSION, items_found_all) - print_specified_property_sorted(TAG_CC_SECURITY_LEVEL, items_found_all) - print_specified_property_sorted(TAG_DEVELOPER, items_found_all) - print_specified_property_sorted(TAG_CERT_LAB, items_found_all) - - -def search_only_headers_bsi(walk_dir): - print('BSI HEADER SEARCH') - LINE_SEPARATOR_STRICT = ' ' - NUM_LINES_TO_INVESTIGATE = 15 - rules_certificate_preface = [ - '(BSI-DSZ-CC-.+?) (?:for|For) (.+?) from (.*)', - '(BSI-DSZ-CC-.+?) zu (.+?) der (.*)', - ] - - items_found_all = {} - items_found = {} - files_without_match = [] - for file_name in search_files(walk_dir): - if not os.path.isfile(file_name): - continue - file_ext = file_name[file_name.rfind('.'):] - if file_ext != '.txt': - continue - # print('*** {} ***'.format(file_name)) - - no_match_yet = True - # - # Process front page with info: cert_id, certified_item and developer - # - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file( - file_name, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT) - - for rule in rules_certificate_preface: - rule_and_sep = rule + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text): - if no_match_yet: - items_found_all[file_name] = {} - items_found_all[file_name] = {} - items_found = items_found_all[file_name] - items_found[TAG_HEADER_MATCH_RULES] = [] - no_match_yet = False - - # insert rule if at least one match for it was found - if rule not in items_found[TAG_HEADER_MATCH_RULES]: - items_found[TAG_HEADER_MATCH_RULES].append(rule) - - match_groups = m.groups() - cert_id = match_groups[0] - certified_item = match_groups[1] - developer = match_groups[2] - - FROM_KEYWORD_LIST = [' from ', ' der '] - for from_keyword in FROM_KEYWORD_LIST: - from_keyword_len = len(from_keyword) - if certified_item.find(from_keyword) != -1: - print( - 'string **{}** detected in certified item - shall not be here, fixing...'.format( - from_keyword)) - certified_item_first = certified_item[:certified_item.find( - from_keyword)] - developer = certified_item[certified_item.find( - from_keyword) + from_keyword_len:] - certified_item = certified_item_first - continue - - end_pos = developer.find('\f-') - if end_pos == -1: - end_pos = developer.find('\fBSI') - if end_pos == -1: - end_pos = developer.find('Bundesamt') - if end_pos != -1: - developer = developer[:end_pos] - - items_found[TAG_CERT_ID] = normalize_match_string(cert_id) - items_found[TAG_CERT_ITEM] = normalize_match_string( - certified_item) - items_found[TAG_DEVELOPER] = normalize_match_string(developer) - items_found[TAG_CERT_LAB] = 'BSI' - - # - # Process page with more detailed certificate info - # PP Conformance, Functionality, Assurance - rules_certificate_third = [ - 'PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified', - ] - - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file( - file_name) - - for rule in rules_certificate_third: - rule_and_sep = rule + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text): - # check if previous rules had at least one match - if not TAG_CERT_ID in items_found.keys(): - print('ERROR: front page not found for file: {}'.format(file_name)) - - match_groups = m.groups() - ref_protection_profiles = match_groups[0] - cc_version = match_groups[1] - cc_security_level = match_groups[2] - - items_found[TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( - ref_protection_profiles) - items_found[TAG_CC_VERSION] = normalize_match_string( - cc_version) - items_found[TAG_CC_SECURITY_LEVEL] = normalize_match_string( - cc_security_level) - - if no_match_yet: - files_without_match.append(file_name) - - if False: - print_found_properties(items_found_all) - - with open("certificate_data_bsiheader.json", "w", errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps(items_found_all, indent=4, sort_keys=True)) - - print('\n*** Certificates without detected preface:') - for file_name in files_without_match: - print('No hits for {}'.format(file_name)) - print('Total no hits files: {}'.format(len(files_without_match))) - print('\n**********************************') - - return items_found_all, files_without_match - - -def search_only_headers_anssi(walk_dir): - class HEADER_TYPE(Enum): - HEADER_FULL = 1 - HEADER_MISSING_CERT_ITEM_VERSION = 2 - HEADER_MISSING_PROTECTION_PROFILES = 3 - HEADER_DUPLICITIES = 4 - - rules_certificate_preface = [ - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)()Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur\(s\)(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom des produits(.+)Référence/version des produits(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d\’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit \(référence/version\)(.+)Nom de la TOE \(référence/version\)(.+)Conformité à un profil de protection(.+)Critères d\’évaluation et version(.+)Niveau d\’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur\(s\)(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit \(référence/version\)(.+)Nom de la TOE \(référence/version\)(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à des profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit \(référence/version\)(.+)Nom de la TOE \(référence/version\)(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Certification Report(.+)Nom du produit(.+)Référence/version du produit(.*)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profisl de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centres d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Version du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Conformité aux profils de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur\(s\)(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Versions du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeur (.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence du produit(.+)Conformité à un profil de protection(.+)Critères d’évaluation et version(.+)Niveau d’évaluation(.+)Développeurs(.+)Centre d’évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Mutual Recognition Agreements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name(.+)Product reference(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer\(s\)(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Products names(.+)Products references(.+)protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)Product name \(reference / version\)(.+)TOE name \(reference / version\)(.+)Protection profile conformity(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developers(.+)Evaluation facility(.+)Recognition arrangements'), - (HEADER_TYPE.HEADER_FULL, - 'Certification report reference(.+)TOE name(.+)Product\'s reference/ version(.+)TOE\'s reference/ version(.+)Conformité à un profil de protection(.+)Evaluation criteria and version(.+)Evaluation level(.+)Developer (.+)Evaluation facility(.+)Recognition arrangements'), - - # corrupted text (duplicities) - (HEADER_TYPE.HEADER_DUPLICITIES, - 'Référencce du rapport de d certification n(.+)Nom du p produit(.+)Référencce/version du produit(.+)Conformiité à un profil de d protection(.+)Critères d d’évaluation ett version(.+)Niveau d’’évaluation(.+)Développ peurs(.+)Centre d’’évaluation(.+)Accords d de reconnaisssance applicab bles'), - - # rules without product version - (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - 'Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - 'Référence du rapport de certification(.+)Nom et version du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeur (.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - (HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION, - 'Référence du rapport de certification(.+)Nom du produit(.+)Conformité à un profil de protection(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - - # rules without protection profile - (HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES, - 'Référence du rapport de certification(.+)Nom du produit(.+)Référence/version du produit(.+)Critères d\'évaluation et version(.+)Niveau d\'évaluation(.+)Développeurs(.+)Centre d\'évaluation(.+)Accords de reconnaissance applicables'), - ] - - # rules_certificate_preface = [ - # (HEADER_TYPE.HEADER_FULL, 'ddddd'), - # ] - - # statistics about rules success rate - num_rules_hits = {} - for rule in rules_certificate_preface: - num_rules_hits[rule[1]] = 0 - - print('***ANSSI HEADER SEARCH***') - items_found_all = {} - files_without_match = [] - for file_name in search_files(walk_dir): - if not os.path.isfile(file_name): - continue - file_ext = file_name[file_name.rfind('.'):] - if file_ext != '.txt': - continue - # print('*** {} ***'.format(file_name)) - - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file( - file_name) - - # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs - pos = whole_text.find(' ') - if pos != -1: - pos = whole_text.find(' ', pos) - if pos != -1: - whole_text = whole_text[pos:] - - no_match_yet = True - other_rule_already_match = False - other_rule = '' - rule_index = -1 - for rule in rules_certificate_preface: - rule_index += 1 - rule_and_sep = rule[1] + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text): - if no_match_yet: - items_found_all[file_name] = {} - items_found_all[file_name] = {} - items_found = items_found_all[file_name] - items_found[TAG_HEADER_MATCH_RULES] = [] - no_match_yet = False - - # insert rule if at least one match for it was found - if rule not in items_found[TAG_HEADER_MATCH_RULES]: - items_found[TAG_HEADER_MATCH_RULES].append(rule[1]) - - if not other_rule_already_match: - other_rule_already_match = True - other_rule = rule - else: - print( - 'WARNING: multiple rules are matching same certification document: ' + file_name) - - num_rules_hits[rule[1]] += 1 # add hit to this rule - - match_groups = m.groups() - - index_next_item = 0 - - items_found[TAG_CERT_ID] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - items_found[TAG_CERT_ITEM] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION: - items_found[TAG_CERT_ITEM_VERSION] = '' - else: - items_found[TAG_CERT_ITEM_VERSION] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES: - items_found[TAG_REFERENCED_PROTECTION_PROFILES] = '' - else: - items_found[TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - items_found[TAG_CC_VERSION] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - items_found[TAG_CC_SECURITY_LEVEL] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - items_found[TAG_DEVELOPER] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - items_found[TAG_CERT_LAB] = normalize_match_string( - match_groups[index_next_item]) - index_next_item += 1 - - if no_match_yet: - files_without_match.append(file_name) - - if False: - print_found_properties(items_found_all) - - # store results into file with fixed name and also with time appendix - with open("certificate_data_anssiheader.json", "w", errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps(items_found_all, indent=4, sort_keys=True)) - - print('\n*** Certificates without detected preface:') - for file_name in files_without_match: - print('No hits for {}'.format(file_name)) - print('Total no hits files: {}'.format(len(files_without_match))) - print('\n**********************************') - - if True: - print('# hits for rule') - sorted_rules = sorted(num_rules_hits.items(), - key=operator.itemgetter(1), reverse=True) - used_rules = [] - for rule in sorted_rules: - print('{:4d} : {}'.format(rule[1], rule[0])) - if rule[1] > 0: - used_rules.append(rule[0]) - - return items_found_all, files_without_match - - -def extract_certificates_frontpage(walk_dir, write_output_file=True): - anssi_items_found, anssi_files_without_match = search_only_headers_anssi( - walk_dir) - bsi_items_found, bsi_files_without_match = search_only_headers_bsi( - walk_dir) - - print('*** Files without detected header') - files_without_match = list( - set(anssi_files_without_match) & set(bsi_files_without_match)) - for file_name in files_without_match: - print(file_name) - print('Total no hits files: {}'.format(len(files_without_match))) - - items_found_all = {**anssi_items_found, **bsi_items_found} - # store results into file with fixed name and also with time appendix - - if write_output_file: - with open("certificate_data_frontpage_all.json", "w", errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps( - items_found_all, indent=4, sort_keys=True)) - - return items_found_all - - -def search_pp_only_headers(walk_dir): - # LINE_SEPARATOR_STRICT = ' ' - # NUM_LINES_TO_INVESTIGATE = 15 - # rules_certificate_preface = [ - # '(Common Criteria Protection Profile .+)?(BSI-PP-CC-.+?) Federal Office for Information Security', - # '(Protection Profile for the .+)?Schutzprofil für das .+?Certification-ID (BSI-CC-PP-[0-9]+?) ', - # # 'Protection Profile for the Security Module of a Smart Meter Mini-HSM (Mini-HSM Security Module PP) Schutzprofil für das Sicherheitsmodul des Smart Meter Mini-HSM Mini-HSM SecMod-PP Version 1.0 – 23 June 2017 Certification-ID BSI-CC-PP-0095 Mini-HSM Security Module PP Bundesamt für Sicherheit in der Informationstechnik' - # ] - - class HEADER_TYPE(Enum): - BSI_TYPE1 = 1 - BSI_TYPE2 = 2 - - DCSSI_TYPE1 = 11 - DCSSI_TYPE2 = 12 - FRONT_DCSSI_TYPE3 = 13 - FRONT_DCSSI_TYPE4 = 14 - DCSSI_TYPE5 = 15 - DCSSI_TYPE6 = 16 - - ANSSI_TYPE1 = 41 - ANSSI_TYPE2 = 42 - ANSSI_TYPE3 = 43 - - rules_pp_third = [ - (HEADER_TYPE.BSI_TYPE1, - 'PP Reference .+?Title (.+)?CC Version (.+)?Assurance Level (.+)?General Status (.+)?Version Number (.+)?Registration (.+)?Keywords (.+)?TOE Overview'), - (HEADER_TYPE.BSI_TYPE2, - 'PP Reference.+?Title: (.+)?Version: (.+)?Date: (.+)?Authors: (.+)?Registration: (.+)?Certification-ID: (.+)?Evaluation Assurance Level: (.+)?CC Version: (.+)?Keywords: (.+)?Specific Terms'), - (HEADER_TYPE.ANSSI_TYPE1, - 'PROTECTION PROFILE IDENTIFICATION.+?Title: (.+)?Version: (.+)?Publication date: (.+)?Certified by: (.+)?Sponsor: (.+)?Editor: (.+)?Review Committee: (.+)?This Protection Profile is conformant to the Common Criteria version (.+)?The minimum assurance level for this Protection Profile is (.+)?PROTECTION PROFILE PRESENTATION'), - (HEADER_TYPE.ANSSI_TYPE2, - 'PP reference.+?Title : (.+)?Version : (.+)?Authors : (.+)?Evaluation Assurance Level : (.+)?Registration : (.+)?Conformant to Version (.+)?of Common Criteria.+?Key words : (.+)?A glossary of terms'), - (HEADER_TYPE.ANSSI_TYPE3, - 'Introduction.+?Title: (.+)?Identifications: (.+)?Editor: (.+)?Date: (.+)?Version: (.+)?Sponsor: (.+)?CC Version: (.+)? This Protection Profile'), - (HEADER_TYPE.DCSSI_TYPE1, - 'Protection profile reference[ ]*Title: (.+)?Reference: (.+)?, Version (.+)?, (.+)?Author: (.+)?Context'), - (HEADER_TYPE.DCSSI_TYPE2, - 'Protection profile reference[ ]*Title: (.+)?Author: (.+)?Version: (.+)?Context'), - (HEADER_TYPE.FRONT_DCSSI_TYPE3, - 'Direction centrale de la sécurité des systèmes d\’information(.+)?(?:Creation date|Date)[ ]*[:]*(.+)?Reference[ ]*[:]*(.+)?Version[ ]*[:]*(.+)?Courtesy Translation[ ]*Courtesy translation.+?under the reference (DCSSI-PP-[0-9/]+)?\.[ ]*Page'), - # (HEADER_TYPE.FRONT_DCSSI_TYPE4, - # 'Direction centrale de la sécurité des systèmes d\’information(.+)?Date[ ]*:(.+)?Reference[ ]*:(.+)?Version[ ]*:(.+)?Courtesy Translation[ ]*Courtesy translation.+?under the reference (DCSSI-PP-[0-9/]+)?\.[ ]*Page'), - (HEADER_TYPE.FRONT_DCSSI_TYPE4, - 'Direction centrale de la sécurité des systèmes d’information (.+)?(?:Creation date|Date)[ ]*:(.+)?Reference[ ]*:(.+)?Version[ ]*:(.+)?Courtesy Translation[ ]*Courtesy translation.+?under the reference (DCSSI-PP-[0-9/]+)?\.[ ]*Page'), - # 'Direction centrale de la sécurité des systèmes d’information Time-stamping System Protection Profile Date : July 18, 2008 Reference : PP-SH-CCv3.1 Version : 1.7 Courtesy Translation Courtesy translation of the protection profile registered and certified by the French Certification Body under the reference DCSSI-PP-2008/07. Page' - (HEADER_TYPE.DCSSI_TYPE5, - 'Protection Profile identification[ ]*Title[ ]*[:]*(.+)?Author[ ]*[:]*(.+)?Version[ ]*[:]*(.+)?,(.+)?Sponsor[ ]*[:]*(.+)?CC version[ ]*[:]*(.+)?(?:Context|Protection Profile introduction)'), - (HEADER_TYPE.DCSSI_TYPE6, - 'PP reference.+?Title[ ]*:(.+)?Author[ ]*:(.+)?Version[ ]*:(.+)?Date[ ]*:(.+)?Sponsor[ ]*:(.+)?CC version[ ]*:(.+)?This protection profile.+?The evaluation assurance level required by this protection profile is (.+)?specified by the DCSSI qualification process'), - # (HEADER_TYPE.DCSSI_TYPE7, - # 'Protection Profile identification.+?Title[ ]*[:]*(.+)?Author[ ]*[:]*(.+)?Version[ ]*[:]*(.+)?,(.+)?Sponsor[ ]*[:]*(.+)?CC version[ ]*[:]*(.+)?Protection Profile introduction') - ] - print("***PP HEADER SEARCH***") - items_found_all = {} - files_without_match = [] - for file_name in search_files(walk_dir): - if not os.path.isfile(file_name): - continue - file_ext = file_name[file_name.rfind('.'):] - if file_ext != '.txt': - continue - # print('*** {} ***'.format(file_name)) - - # - # Process page with more detailed protection profile info - # PP Reference - - whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file( - file_name) - - no_match_yet = True - for rule in rules_pp_third: - rule_and_sep = rule[1] + REGEXEC_SEP - - for m in re.finditer(rule_and_sep, whole_text): - if no_match_yet: - items_found_all[file_name] = {} - items_found_all[file_name] = {} - items_found = items_found_all[file_name] - items_found[TAG_HEADER_MATCH_RULES] = [] - no_match_yet = False - - # insert rule if at least one match for it was found - if rule[1] not in items_found[TAG_HEADER_MATCH_RULES]: - items_found[TAG_HEADER_MATCH_RULES].append(rule[1]) - - match_groups = m.groups() - index = 0 - - if rule[0] == HEADER_TYPE.BSI_TYPE1: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_VERSION, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_SECURITY_LEVEL, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_GENERAL_STATUS, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_ID, - normalize_match_string(match_groups[index])) - index += 1 - keywords = match_groups[index].lstrip(' ') - set_match_string(items_found, TAG_KEYWORDS, normalize_match_string( - keywords[0:keywords.find(' ')])) - index += 1 - set_match_string(items_found, TAG_PP_AUTHORS, 'BSI') - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'BSI') - - if rule[0] == HEADER_TYPE.BSI_TYPE2: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_DATE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_AUTHORS, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_REGISTRATOR, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_ID, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_SECURITY_LEVEL, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_VERSION, - normalize_match_string(match_groups[index])) - index += 1 - keywords = match_groups[index].lstrip(' ') - set_match_string(items_found, TAG_KEYWORDS, normalize_match_string( - keywords[0:keywords.find(' ')])) - index += 1 - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'BSI') - - if rule[0] == HEADER_TYPE.ANSSI_TYPE1: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_DATE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_REGISTRATOR, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_SPONSOR, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_EDITOR, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_REVIEWER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_VERSION, - normalize_match_string(match_groups[index])) - index += 1 - level = match_groups[index].lstrip(' ') - set_match_string(items_found, TAG_CC_SECURITY_LEVEL, normalize_match_string( - level[0:level.find(' ')])) - index += 1 - - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI') - - if rule[0] == HEADER_TYPE.ANSSI_TYPE2: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_AUTHORS, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_SECURITY_LEVEL, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_REGISTRATOR, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_VERSION, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_KEYWORDS, - normalize_match_string(match_groups[index])) - index += 1 - - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI') - - if rule[0] == HEADER_TYPE.ANSSI_TYPE3: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - # todo: parse if multiple pp ids are present - set_match_string(items_found, TAG_PP_ID, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_EDITOR, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_DATE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_SPONSOR, - normalize_match_string(match_groups[index])) - index += 1 - ccversion = match_groups[index].lstrip(' ') - set_match_string(items_found, TAG_CC_VERSION, normalize_match_string( - ccversion[0:ccversion.find(' ')])) - index += 1 - - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI') - - if rule[0] == HEADER_TYPE.DCSSI_TYPE1: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_ID, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_DATE, - normalize_match_string(match_groups[index])) - index += 1 - author = match_groups[index].lstrip(' ') - set_match_string(items_found, TAG_PP_AUTHORS, normalize_match_string( - author[0:author.find(' ')])) - index += 1 - - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI') - - if rule[0] == HEADER_TYPE.DCSSI_TYPE2: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_AUTHORS, - normalize_match_string(match_groups[index])) - index += 1 - version = match_groups[index].lstrip(' ') - set_match_string(items_found, TAG_PP_VERSION_NUMBER, normalize_match_string( - version[0:version.find(' ')])) - index += 1 - - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI') - - if rule[0] == HEADER_TYPE.FRONT_DCSSI_TYPE3 or rule[0] == HEADER_TYPE.FRONT_DCSSI_TYPE4: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_DATE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_ID, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_ID_REGISTRATOR, - normalize_match_string(match_groups[index])) - index += 1 - - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI') - - if rule[0] == HEADER_TYPE.DCSSI_TYPE5: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_AUTHORS, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_DATE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_SPONSOR, - normalize_match_string(match_groups[index])) - index += 1 - ccversion = match_groups[index].lstrip(' ') - set_match_string(items_found, TAG_CC_VERSION, normalize_match_string( - ccversion[0:ccversion.find(' ')])) - index += 1 - - if rule[0] == HEADER_TYPE.DCSSI_TYPE6: - set_match_string(items_found, TAG_PP_TITLE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_AUTHORS, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_VERSION_NUMBER, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_DATE, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_PP_SPONSOR, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_VERSION, - normalize_match_string(match_groups[index])) - index += 1 - set_match_string(items_found, TAG_CC_SECURITY_LEVEL, - normalize_match_string(match_groups[index])) - index += 1 - - set_match_string( - items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI') - - if no_match_yet: - files_without_match.append(file_name) - - if False: - print_found_properties(items_found_all) - - with open("pp_data_header.json", "w", errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps(items_found_all, indent=4, sort_keys=True)) - - print('\n*** Protection profiles without detected header:') - for file_name in files_without_match: - print('No hits for {}'.format(file_name)) - print('Total no hits files: {}'.format(len(files_without_match))) - print('\n**********************************') - - return items_found_all, files_without_match - - -def extract_protectionprofiles_frontpage(walk_dir, write_output_file=True): - pp_items_found, pp_files_without_match = search_pp_only_headers(walk_dir) - - print('*** Files without detected protection profiles header') - for file_name in pp_files_without_match: - print(file_name) - print('Total no hits files: {}'.format(len(pp_files_without_match))) - - # store results into file with fixed name and also with time appendix - if write_output_file: - with open("pp_data_frontpage_all.json", "w", errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps( - pp_items_found, indent=4, sort_keys=True)) - - return pp_items_found - - -def extract_certificates_keywords(walk_dir, fragments_dir, file_prefix, write_output_file=True, - should_censure_right_away=False, fips_items=None): - # ensure existence of fragments folder - if not os.path.exists(fragments_dir): - os.makedirs(fragments_dir) - print("***EXTRACT KEYWORDS***") - all_items_found = {} - # cert_id = {} - for file_name in search_files(walk_dir): - if not os.path.isfile(file_name): - continue - file_ext = file_name[file_name.rfind('.'):] - if file_ext != '.txt': - continue - - # print('*** {} ***'.format(file_name)) - - file_cert_name = os.path.splitext( - os.path.splitext(os.path.basename(file_name))[0])[0] - # parse certificate, return all matches - all_items_found[file_name], modified_cert_file = parse_cert_file( - file_name, fips_rules if fips_items else rules, -1, should_censure_right_away=should_censure_right_away, - fips_items=fips_items) - - # try to establish the certificate id of the current certificate - # cert_id[file_cert_name] = estimate_cert_id( - # None, all_items_found[file_cert_name], file_name) - - # save report text with highlighted/replaced matches into \\fragments\\ directory - base_path = file_name[:file_name.rfind(os.sep)] - file_name_short = file_name[file_name.rfind(os.sep) + 1:] - target_file = '{}{}{}'.format(fragments_dir, os.sep, file_name_short) - save_modified_cert_file( - target_file, modified_cert_file[0], modified_cert_file[1]) - - # store results into file with fixed name - if write_output_file: - with open("{}_data_keywords_all.json".format(file_prefix), "w", errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps( - all_items_found, indent=4, sort_keys=True)) - - # print('\nTotal matches found in separate files:') - # print_total_matches_in_files(all_items_found_count) - - # print('\nFile name and estimated certificate ID:') - # print_guessed_cert_id(cert_id) - - # depricated_print_dot_graph_keywordsonly(['rules_cert_id'], all_items_found, cert_id, walk_dir, 'certid_graph_from_keywords.dot', True) - - total_items_found = 0 - for file_name in all_items_found: - total_items_found += count_num_items_found(all_items_found[file_name]) - - PRINT_MATCHES = False - if PRINT_MATCHES: - all_matches = [] - for file_name in all_items_found: - print('*' * 10, "FILENAME:", file_name, '*' * 10) - for rule_group in all_items_found[file_name].keys(): - items_found = all_items_found[file_name][rule_group] - for rule in items_found.keys(): - for match in items_found[rule]: - if match not in all_matches: - print(match) - # all_matches.append(match) - - sorted_all_matches = sorted(all_matches) - # for match in sorted_all_matches: - # print(match) - - # verify total matches found - print('\nTotal matches found: {}'.format(total_items_found)) - - return all_items_found - - -def extract_certificates_pdfmeta(walk_dir, file_prefix, write_output_file=True): - all_items_found = {} - counter = 0 - for file_name in search_files(walk_dir): - if not os.path.isfile(file_name): - continue - file_ext = file_name[file_name.rfind('.'):] - if file_ext != '.pdf': - continue - - print("***EXTRACT PDFMETA***") - # print('*** {} ***'.format(file_name)) - - item = {} - item['pdf_file_size_bytes'] = os.path.getsize(file_name) - try: - with open(file_name, 'rb') as f: - pdf = PdfFileReader(f) - # store additional interesting info - item['pdf_is_encrypted'] = pdf.getIsEncrypted() - item['pdf_number_of_pages'] = pdf.getNumPages() - - # extract pdf metadata (as dict) and save it - info = pdf.getDocumentInfo() - if info is not None: - for key in info: - item[key] = str(info[key]) - except Exception as e: - item['error'] = str(e) - - # test save of the data extracted to prevent error only very later - # try: - # with open("{}_temp.json".format(file_prefix), "w") as write_file: - # write_file.write(json.dumps(item, indent=4, sort_keys=True)) - # except Exception: - # print(' ERROR: invalid data from pdf') - - all_items_found[file_name] = item - - if counter % 100 == 0: - # store results into file with fixed name - with open("{}_data_pdfmeta_{}.json".format(file_prefix, counter), "w", - errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps( - all_items_found, indent=4, sort_keys=True)) - counter += 1 - - # store allresults into file with fixed name - if write_output_file: - with open("{}_data_pdfmeta_all.json".format(file_prefix), "w", errors=FILE_ERRORS_STRATEGY) as write_file: - write_file.write(json.dumps( - all_items_found, indent=4, sort_keys=True)) - - return all_items_found - - -def extract_file_name_from_url(url): - file_name = url[url.rfind('/') + 1:] - file_name = file_name.replace('%20', ' ') - return file_name - - -def parse_product_updates(updates_chunk, link_files_updates): - maintenance_reports = [] - - rule_with_maintainance_ST = '.*?([0-9]+?-[0-9]+?-[0-9]+?) (.+?)\
') - eal_level = security_level - eal_augmented = [] - if start_pos != -1: - eal_level = normalize_match_string(security_level[:start_pos]) - # some augmented items found - augm_chunk = security_level[start_pos:] - augm_chunk += ' ' - # items are in form of
AVA_VLA.4
AVA_MSU.3 ... - rule = '\(.+?) ' - - for m in re.finditer(rule, augm_chunk): - match_groups = m.groups() - eal_augmented.append(normalize_match_string(match_groups[0])) - - return eal_level, eal_augmented - - -def extract_certificates_metadata_html(file_name): - print("***HTML METADATA***") - print(file_name) - items_found_all = {} - download_files_certs = [] - download_files_updates = [] - # print('*** {} ***'.format(file_name)) - - whole_text = load_cert_html_file(file_name) - - whole_text = whole_text.replace('\n', ' ') - whole_text = whole_text.replace(' ', ' ') - whole_text = whole_text.replace('&', '&') - - # First find end extract chunks between ... - start_pos = whole_text.find('', start_pos + 1) - odd_start_pos = whole_text.find('', start_pos + 1) - - start_pos = min(even_start_pos, odd_start_pos) - - # skip chunks which are not cert item chunks - if chunk.find('This list was generated on') != -1: - continue - - chunks_found += 1 - - class HEADER_TYPE(Enum): - HEADER_FULL = 1 - HEADER_MISSING_VENDOR_WEB = 2 - - # IMPORTANT: order regexes based on their specificity - the most specific goes first - rules_cc_html = [ - (HEADER_TYPE.HEADER_FULL, - '\[ ]+\(.+?)\(.+?) -->' - '.+?\Certification Report\' - '.+?\Security Target' - '.+?\