Merge branch 'master' into fips

5 files changed, 743 insertions, 521 deletions

diff --git a/process_certificates.py b/process_certificates.py
index 143c6be2..2b63d70a 100755
--- a/process_certificates.py
+++ b/process_certificates.py
@@ -45,7 +45,7 @@ def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_ext
     #
     # Start processing
     #
-    do_analysis_filtered = False
+    do_analysis_filtered = True
 
     if do_complete_extraction:
         # analyze all files from scratch, set 'previous' state to empty dict
@@ -113,18 +113,19 @@ def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_ext
         convert_pdf_files(walk_dir, threads, ["-raw"])
 
     if do_extraction_certs:
-        all_front = extract_certificates_frontpage(walk_dir)
-        all_keywords = extract_certificates_keywords(walk_dir, fragments_dir, 'certificate')
-        all_pdf_meta = extract_certificates_pdfmeta(walk_dir, 'certificate', results_dir)
+        all_keywords = extract_certificates_keywords_parallel(walk_dir, fragments_dir, 'certificate', threads)
+        with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file:
+            json.dump(all_keywords,  write_file, indent=4, sort_keys=True)
 
-        # save joined results
+        all_front = extract_certificates_frontpage(walk_dir)
         with open(results_dir / "certificate_data_frontpage_all.json", "w") as write_file:
             json.dump(all_front,  write_file, indent=4, sort_keys=True)
-        with open(results_dir / "certificate_data_keywords_all.json", "w") as write_file:
-            json.dump(all_keywords,  write_file, indent=4, sort_keys=True)
+
+        all_pdf_meta = extract_certificates_pdfmeta_parallel(walk_dir, 'certificate', threads)
         with open(results_dir / "certificate_data_pdfmeta_all.json", "w") as write_file:
             json.dump(all_pdf_meta,  write_file, indent=4, sort_keys=True)
 
+
     # if do_extraction_pp:
     #     all_pp_csv = extract_protectionprofiles_csv(web_dir)
     #     all_pp_front = extract_protectionprofiles_frontpage(pp_dir)
@@ -174,6 +175,7 @@ def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_ext
 
     if do_processing:
         # load information about protection profiles as extracted by sec-certs-pp tool
+        all_pp_items = {}
         with open(results_dir / 'pp_data_complete_processed.json') as json_file:
             all_pp_items = json.load(json_file)
 
@@ -190,6 +192,9 @@ def main(directory, do_complete_extraction: bool, do_download_meta: bool, do_ext
             all_cert_items = json.load(json_file)
 
         if do_analysis_filtered:
+            # plot only selected analysis up to date 2020
+            do_analysis_force_end_date(all_cert_items, results_dir, 2020)
+
             # analyze only smartcards
             do_analysis_only_filtered(all_cert_items, results_dir,
                                       ['csv_scan', 'cc_category'], 'ICs, Smart Cards and Smart Card-Related Devices and Systems')
diff --git a/sec_certs/analyze_certificates.py b/sec_certs/analyze_certificates.py
index db907ce4..6f1dce72 100644
--- a/sec_certs/analyze_certificates.py
+++ b/sec_certs/analyze_certificates.py
@@ -6,6 +6,7 @@ from pathlib import Path
 
 import numpy as np
 import matplotlib.pyplot as plt
+import copy
 
 from matplotlib.pyplot import figure
 from dateutil import parser
@@ -432,7 +433,7 @@ def plot_schemes_multi_line_graph(x_ticks, data, prominent_data, x_label, y_labe
         if group in prominent_data:
             plt.plot(x_ticks, items_in_year, line_types[num_lines_plotted % len(line_types)], label=group, linewidth=3)
         else:
-            # plot minor suppliers dashed
+            # plot non-prominent data as dashed
             plt.plot(x_ticks, items_in_year, line_types[num_lines_plotted % len(line_types)], label=group, linewidth=2)
 
         # change line type to prevent color repetitions
@@ -449,7 +450,20 @@ def plot_schemes_multi_line_graph(x_ticks, data, prominent_data, x_label, y_labe
     plt.close()
 
 
-def analyze_cert_years_frequency(all_cert_items, filter_label):
+def filter_end_year(items: dict, end_year: int):
+    filtered_items = {}
+
+    for item in items.keys():
+        filtered_items[item] = {}
+        for year in items[item]:
+            # copy only years below end_year
+            if year <= end_year:
+                filtered_items[item][year] = copy.deepcopy(items[item][year])
+
+    return filtered_items
+
+
+def analyze_cert_years_frequency(all_cert_items, filter_label, force_plot_end_year=None):
     scheme_date = {}
     level_date = {}
     category_date = {}
@@ -583,6 +597,7 @@ def analyze_cert_years_frequency(all_cert_items, filter_label):
                         # certificate is valid in year
                         valid_in_years['active'][year].append(cert_long_id)
 
+
     # print manufacturers frequency
     sorted_by_occurence = sorted(manufacturer_items.items(), key=operator.itemgetter(1))
     print('\n### Frequency of certificates per company')
@@ -593,35 +608,48 @@ def analyze_cert_years_frequency(all_cert_items, filter_label):
     for manufacturer in sorted_by_occurence:
         print('  {}: {}x'.format(manufacturer[0], manufacturer[1]))
 
-    # plot graphs showing cert. scheme and EAL in years
-    years = np.arange(START_YEAR, END_YEAR)
-    years_extended = np.arange(START_YEAR, END_YEAR + ARCHIVE_OFFSET)
-    plot_schemes_multi_line_graph(years, scheme_date, ['DE', 'JP', 'FR', 'US', 'CA'], 'Year of issuance', 'Number of certificates issued', fig_label('CC certificates issuance frequency per scheme and year', filter_label), 'num_certs_in_years')
-    plot_schemes_multi_line_graph(years, level_date, ['EAL4+', 'EAL5+','EAL2+', 'Protection Profile'], 'Year of issuance', 'Number of certificates issued', fig_label('Certificates issuance frequency per EAL and year', filter_label), 'num_certs_eal_in_years')
-    plot_schemes_multi_line_graph(years, category_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Category of certificates issued in given year', filter_label), 'num_certs_category_in_years')
-    plot_schemes_multi_line_graph(years, pp_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Certificates with/without conforming to Protection Profile', filter_label), 'num_certs_pp_in_years')
-    plot_schemes_multi_line_graph(years, labs_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Number of certificates certified by laboratory in given year', filter_label), 'num_certs_by_lab_in_years')
-    plot_schemes_multi_line_graph(years_extended, archive_date, [], 'Year of issuance', 'Number of certificates', fig_label('Number of certificates archived or planned for archival in a given year', filter_label), 'num_certs_archived_in_years')
-    plot_schemes_multi_line_graph(years_extended, valid_in_years, [], 'Year', 'Number of certificates', fig_label('Number of certificates active and archived in given year', filter_label), 'num_certs_active_archived_in_years')
+    # plot only top manufacturers
+    top_manufacturers = dict(sorted_by_occurence[len(sorted_by_occurence) - 20:]).keys()  # top 20 manufacturers
+    top_manufacturers_date = {}
+    for manuf in manufacturer_date.keys():
+        if manuf in top_manufacturers:
+            top_manufacturers_date[manuf] = manufacturer_date[manuf]
+
+    # filter only subset of years if required
+    if force_plot_end_year:
+        years = np.arange(START_YEAR, force_plot_end_year + 1)
+        plot_scheme_date = filter_end_year(scheme_date, force_plot_end_year)
+        plot_level_date = filter_end_year(level_date, force_plot_end_year)
+        plot_category_date = filter_end_year(category_date, force_plot_end_year)
+        plot_pp_date = filter_end_year(pp_date, force_plot_end_year)
+        plot_labs_date = filter_end_year(labs_date, force_plot_end_year)
+        plot_top_manufacturers_date = filter_end_year(top_manufacturers_date, force_plot_end_year)
+    else:
+        # plot all
+        years = np.arange(START_YEAR, END_YEAR)
+        plot_scheme_date = scheme_date
+        plot_level_date = level_date
+        plot_category_date = category_date
+        plot_pp_date = pp_date
+        plot_labs_date = labs_date
+        plot_top_manufacturers_date = top_manufacturers_date
 
     sc_manufacturers = ['Gemalto', 'NXP Semiconductors', 'Samsung', 'STMicroelectronics', 'Oberthur Technologies',
                         'Infineon Technologies AG', 'G+D Mobile Security GmbH', 'ATMEL Smart Card ICs', 'Idemia',
                         'Athena Smartcard', 'Renesas', 'Philips Semiconductors GmbH', 'Oberthur Card Systems']
 
-    # plot only top manufacturers
-    top_manufacturers = dict(sorted_by_occurence[len(sorted_by_occurence) - 20:]).keys()  # top 20 manufacturers
-    plot_manufacturers_date = {}
-    for manuf in manufacturer_date.keys():
-        if manuf in top_manufacturers:
-            plot_manufacturers_date[manuf] = manufacturer_date[manuf]
-    plot_schemes_multi_line_graph(years, plot_manufacturers_date, sc_manufacturers, 'Year of issuance', 'Number of certificates issued', fig_label('Top 20 manufacturers of certified items per year', filter_label), 'manufacturer_in_years')
+    # plot graphs showing cert. scheme and EAL in years
+    plot_schemes_multi_line_graph(years, plot_scheme_date, ['DE', 'JP', 'FR', 'US', 'CA'], 'Year of issuance', 'Number of certificates issued', fig_label('CC certificates issuance frequency per scheme and year', filter_label), 'num_certs_in_years')
+    plot_schemes_multi_line_graph(years, plot_level_date, ['EAL4+', 'EAL5+','EAL2+', 'Protection Profile'], 'Year of issuance', 'Number of certificates issued', fig_label('Certificates issuance frequency per EAL and year', filter_label), 'num_certs_eal_in_years')
+    plot_schemes_multi_line_graph(years, plot_category_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Category of certificates issued in given year', filter_label), 'num_certs_category_in_years')
+    plot_schemes_multi_line_graph(years, plot_pp_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Certificates with/without conforming to Protection Profile', filter_label), 'num_certs_pp_in_years')
+    plot_schemes_multi_line_graph(years, plot_labs_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Number of certificates certified by laboratory in given year', filter_label), 'num_certs_by_lab_in_years')
+    plot_schemes_multi_line_graph(years, plot_top_manufacturers_date, sc_manufacturers, 'Year of issuance', 'Number of certificates issued', fig_label('Top 20 manufacturers of certified items per year', filter_label), 'manufacturer_in_years')
 
-    # plot only smartcard manufacturers
-    plot_manufacturers_date = {}
-    for manuf in manufacturer_date.keys():
-        if manuf in sc_manufacturers:
-            plot_manufacturers_date[manuf] = manufacturer_date[manuf]
-    # plot_schemes_multi_line_graph(years, plot_manufacturers_date, [], 'Year of issuance', 'Number of certificates issued', fig_label('Smartcard-related manufacturers of certified items per year', filter_label), 'manufacturer_sc_in_years')
+    # plot stats with extended range
+    years_extended = np.arange(START_YEAR, END_YEAR + ARCHIVE_OFFSET)
+    plot_schemes_multi_line_graph(years_extended, archive_date, [], 'Year of issuance', 'Number of certificates', fig_label('Number of certificates archived or planned for archival in a given year', filter_label), 'num_certs_archived_in_years')
+    plot_schemes_multi_line_graph(years_extended, valid_in_years, [], 'Year', 'Number of certificates', fig_label('Number of certificates active and archived in given year', filter_label), 'num_certs_active_archived_in_years')
 
     # plot certificate validity lengths
     print('### Certificates validity period lengths:')
@@ -858,6 +886,14 @@ def do_analysis_09_01_2019_archival(all_cert_items, current_dir: Path):
     do_all_analysis(limited_cert_items, 'cc_archived_date={}'.format(archived_date))
 
 
+def do_analysis_force_end_date(all_cert_items, current_dir: Path, force_end_date: int):
+    target_folder = os.path.join(current_dir, 'results_in_years_only_till_{}'.format(force_end_date))
+    if not os.path.exists(target_folder):
+        os.makedirs(target_folder)
+    os.chdir(target_folder)
+#    analyze_cert_years_frequency(all_cert_items, 'forced_end_date={}'.format(force_end_date), force_end_date)
+    analyze_cert_years_frequency(all_cert_items, '', force_end_date)
+
 def do_analysis_manufacturers(all_cert_items, current_dir: Path):
     # analyze only Infineon certificates
     do_analysis_only_filtered(all_cert_items, current_dir,
diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py
index 0fdacae7..16c24893 100644
--- a/sec_certs/cert_rules.py
+++ b/sec_certs/cert_rules.py
@@ -1,3 +1,4 @@
+import copy
 import re
 import json
 
@@ -140,6 +141,45 @@ rules_javacard = [
     r'(?:Java Card|JavaCard) \(version [2-3]\.[0-9](?:\.[0-9]|)\)',
     r'(?:Global Platform|GlobalPlatform) [2-3]\.[0-9]\.[0-9]',
     r'(?:Global Platform|GlobalPlatform) \(version [2-3]\.[0-9]\.[0-9]\)',
+    ]
+
+rules_javacard_api_consts = [
+    # javacard API constants
+    r'ALG_(?:PSEUDO_RANDOM|SECURE_RANDOM|TRNG|ALG_PRESEEDED_DRBG|FAST|KEYGENERATION)',
+    r'ALG_DES_[A-Z_0-9]+',  # may have false positives like XCP_CPB_ALG_EC_BPOOLCRV
+    r'ALG_RSA_[A-Z_0-9]+',
+    r'ALG_DSA_[A-Z_0-9]+',
+    r'ALG_ECDSA_[A-Z_0-9]+',
+    r'ALG_AES_[A-Z_0-9]+',
+    r'ALG_HMAC_[A-Z_0-9]+',
+    r'ALG_KOREAN_[A-Z_0-9]+',
+    r'ALG_EC_[A-Z_0-9]+',
+    r'ALG_SHA_[A-Z_0-9]+',
+    r'ALG_SHA3_[A-Z_0-9]+',
+    r'ALG_MD[A-Z_0-9]+',
+    r'ALG_RIPEMD[A-Z_0-9]+',
+    r'ALG_ISO3309_[A-Z_0-9]+',
+    r'ALG_XDH',
+    r'ALG_SM2',
+    r'ALG_SM3',
+    r'ALG_NULL',
+    r'ALG_TRNG',
+    r'ALG_NULL',
+    r'SIG_CIPHER_[A-Z_0-9]+',
+    r'CIPHER_[A-Z_0-9]+',
+    r'PAD_[A-Z_0-9]+',
+    r'TYPE_[A-Z_0-9]+',
+    r'LENGTH_[A-Z_0-9]+',
+    r'OWNER_PIN[A-Z_0-9]*',
+    # named curves
+    r'BRAINPOOLP[A-Z_0-9]+(?:R|T)1',
+    r'ED25519',
+    r'ED448',
+    r'FRP256V1',
+    r'SECP[0-9]*R1',
+    r'SM2',
+    r'X25519',
+    r'X448',
 ]
 
 rules_crypto_algs = [
@@ -392,6 +432,7 @@ common_rules['rules_security_level'] = rules_security_level
 common_rules['rules_security_assurance_components'] = rules_security_assurance_components
 common_rules['rules_security_functional_components'] = rules_security_functional_components
 common_rules['rules_javacard'] = rules_javacard
+common_rules['rules_javacard_api_consts'] = rules_javacard_api_consts
 common_rules['rules_crypto_algs'] = rules_crypto_algs
 common_rules['rules_block_cipher_modes'] = rules_block_cipher_modes
 common_rules['rules_ecc_curves'] = rules_ecc_curves
@@ -424,7 +465,8 @@ fips_rules = {}
 fips_rules['rules_fips_algorithms'] = rules_fips_remove_algorithm_ids
 fips_rules['rules_security_level'] = rules_fips_security_level
 fips_rules['rules_cert_id'] = rules_fips_cert
-fips_rules.update(common_rules)
+fips_common_rules = copy.deepcopy(common_rules)  # make separate copy not to process cc rules by fips's re.compile
+fips_rules.update(fips_common_rules)
 
 for rule in fips_rules:
     for current_rule in range(len(fips_rules[rule])):
diff --git a/sec_certs/download.py b/sec_certs/download.py
index fc225150..57e332e8 100644
--- a/sec_certs/download.py
+++ b/sec_certs/download.py
@@ -13,8 +13,11 @@ CC_WEB_URL = 'https://www.commoncriteriaportal.org'
 
 def download_file(url: str, output: Path) -> int:
     r = requests.get(url, allow_redirects=True)
-    with output.open("wb") as f:
-        f.write(r.content)
+    try:
+        with open(output, "wb") as f:
+            f.write(r.content)
+    except (OSError, ConnectionError) as e:
+        print('ERROR: Failed to download {} with {}'.format(url, e))
     return r.status_code
 
 
diff --git a/sec_certs/extract_certificates.py b/sec_certs/extract_certificates.py
index e4192855..12054f0a 100644
--- a/sec_certs/extract_certificates.py
+++ b/sec_certs/extract_certificates.py
@@ -5,7 +5,10 @@ import os
 import operator
 
 import subprocess
+from multiprocessing import Pool, RLock
 from multiprocessing.pool import ThreadPool
+from multiprocessing.spawn import freeze_support
+from re import Pattern
 from typing import Sequence
 
 from tqdm import tqdm
@@ -46,17 +49,24 @@ def get_line_number(lines, line_length_compensation, match_start_index):
     return -1
 
 
-def convert_pdf_files(walk_dir: Path, num_threads: int, options: Sequence[str]) -> Sequence[subprocess.CompletedProcess]:
-    def convert_pdf_file(file_name: str):
-        return subprocess.run(["pdftotext", *options, file_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    items = []
+def get_files_to_process(walk_dir: Path, required_extension: str):
+    files_to_process = []
     for file_name in search_files(walk_dir):
         if not os.path.isfile(file_name):
             continue
         file_ext = file_name[file_name.rfind('.'):]
-        if file_ext.lower() != '.pdf':
+        if file_ext.lower() != required_extension:
             continue
-        items.append(file_name)
+        files_to_process.append(file_name)
+
+    return files_to_process
+
+
+def convert_pdf_files(walk_dir: Path, num_threads: int, options: Sequence[str]) -> Sequence[subprocess.CompletedProcess]:
+    def convert_pdf_file(file_name: str):
+        return subprocess.run(["pdftotext", *options, file_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    items = get_files_to_process(walk_dir, '.pdf')
+
     results = []
     with tqdm(total=len(items)) as progress:
         for result in ThreadPool(num_threads).imap(convert_pdf_file, items):
@@ -138,7 +148,7 @@ def set_match_string(items, key_name, new_value):
 
 
 def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=LINE_SEPARATOR,
-                    should_censure_right_away=False, fips_items=None, ):
+                    should_censure_right_away=False, fips_items=None):
     whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
         file_name, limit_max_lines, line_separator)
 
@@ -152,13 +162,17 @@ def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=
         items_found = items_found_all[rule_group]
 
         for rule in search_rules[rule_group]:
-
-            rule_and_sep = rule + REGEXEC_SEP
+            if type(rule) != str:
+                rule_str = rule.pattern
+                rule_and_sep = re.compile(rule.pattern + REGEXEC_SEP)
+            else:
+                rule_str = rule
+                rule_and_sep = rule + REGEXEC_SEP
 
             for m in re.finditer(rule_and_sep, whole_text_with_newlines):
                 # insert rule if at least one match for it was found
                 if rule not in items_found:
-                    items_found[rule] = {}
+                    items_found[rule_str] = {}
 
                 match = m.group()
                 match = normalize_match_string(match)
@@ -178,22 +192,22 @@ def parse_cert_file(file_name, search_rules, limit_max_lines=-1, line_separator=
                 if is_algorithm:
                     continue
 
-                if match not in items_found[rule]:
-                    items_found[rule][match] = {}
-                    items_found[rule][match][TAG_MATCH_COUNTER] = 0
+                if match not in items_found[rule_str]:
+                    items_found[rule_str][match] = {}
+                    items_found[rule_str][match][TAG_MATCH_COUNTER] = 0
                     if APPEND_DETAILED_MATCH_MATCHES:
-                        items_found[rule][match][TAG_MATCH_MATCHES] = []
+                        items_found[rule_str][match][TAG_MATCH_MATCHES] = []
                     # else:
-                    #    items_found[rule][match][TAG_MATCH_MATCHES] = ['List of matches positions disabled. Set APPEND_DETAILED_MATCH_MATCHES to True']
+                    #    items_found[rule_str][match][TAG_MATCH_MATCHES] = ['List of matches positions disabled. Set APPEND_DETAILED_MATCH_MATCHES to True']
 
-                items_found[rule][match][TAG_MATCH_COUNTER] += 1
+                items_found[rule_str][match][TAG_MATCH_COUNTER] += 1
                 match_span = m.span()
                 # estimate line in original text file
                 # line_number = get_line_number(lines, line_length_compensation, match_span[0])
                 # start index, end index, line number
-                # items_found[rule][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number])
+                # items_found[rule_str][match][TAG_MATCH_MATCHES].append([match_span[0], match_span[1], line_number])
                 if APPEND_DETAILED_MATCH_MATCHES:
-                    items_found[rule][match][TAG_MATCH_MATCHES].append(
+                    items_found[rule_str][match][TAG_MATCH_MATCHES].append(
                         [match_span[0], match_span[1]])
                 if should_censure_right_away:
                     whole_text_with_newlines = whole_text_with_newlines.replace(
@@ -358,101 +372,98 @@ def search_only_headers_bsi(walk_dir: Path):
     items_found_all = {}
     items_found = {}
     files_without_match = []
-    for file_name in search_files(walk_dir):
-        if not os.path.isfile(file_name):
-            continue
-        file_ext = file_name[file_name.rfind('.'):]
-        if file_ext != '.txt':
-            continue
-        #        print('*** {} ***'.format(file_name))
+    files_to_process = get_files_to_process(walk_dir, '.txt')
+    with tqdm(total=len(files_to_process)) as progress:
+        for file_name in files_to_process:
+            no_match_yet = True
+            #
+            # Process front page with info: cert_id, certified_item and developer
+            #
+            whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+                file_name, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT)
 
-        no_match_yet = True
-        #
-        # Process front page with info: cert_id, certified_item and developer
-        #
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            file_name, NUM_LINES_TO_INVESTIGATE, LINE_SEPARATOR_STRICT)
+            for rule in rules_certificate_preface:
+                rule_and_sep = rule + REGEXEC_SEP
 
-        for rule in rules_certificate_preface:
-            rule_and_sep = rule + REGEXEC_SEP
+                for m in re.finditer(rule_and_sep, whole_text):
+                    if no_match_yet:
+                        items_found_all[file_name] = {}
+                        items_found_all[file_name] = {}
+                        items_found = items_found_all[file_name]
+                        items_found[TAG_HEADER_MATCH_RULES] = []
+                        no_match_yet = False
 
-            for m in re.finditer(rule_and_sep, whole_text):
-                if no_match_yet:
-                    items_found_all[file_name] = {}
-                    items_found_all[file_name] = {}
-                    items_found = items_found_all[file_name]
-                    items_found[TAG_HEADER_MATCH_RULES] = []
-                    no_match_yet = False
+                    # insert rule if at least one match for it was found
+                    if rule not in items_found[TAG_HEADER_MATCH_RULES]:
+                        items_found[TAG_HEADER_MATCH_RULES].append(rule)
 
-                # insert rule if at least one match for it was found
-                if rule not in items_found[TAG_HEADER_MATCH_RULES]:
-                    items_found[TAG_HEADER_MATCH_RULES].append(rule)
+                    match_groups = m.groups()
+                    cert_id = match_groups[0]
+                    certified_item = match_groups[1]
+                    developer = match_groups[2]
 
-                match_groups = m.groups()
-                cert_id = match_groups[0]
-                certified_item = match_groups[1]
-                developer = match_groups[2]
+                    FROM_KEYWORD_LIST = [' from ', ' der ']
+                    for from_keyword in FROM_KEYWORD_LIST:
+                        from_keyword_len = len(from_keyword)
+                        if certified_item.find(from_keyword) != -1:
+                            print(
+                                'string **{}** detected in certified item - shall not be here, fixing...'.format(
+                                    from_keyword))
+                            certified_item_first = certified_item[:certified_item.find(
+                                from_keyword)]
+                            developer = certified_item[certified_item.find(
+                                from_keyword) + from_keyword_len:]
+                            certified_item = certified_item_first
+                            continue
 
-                FROM_KEYWORD_LIST = [' from ', ' der ']
-                for from_keyword in FROM_KEYWORD_LIST:
-                    from_keyword_len = len(from_keyword)
-                    if certified_item.find(from_keyword) != -1:
-                        print(
-                            'string **{}** detected in certified item - shall not be here, fixing...'.format(
-                                from_keyword))
-                        certified_item_first = certified_item[:certified_item.find(
-                            from_keyword)]
-                        developer = certified_item[certified_item.find(
-                            from_keyword) + from_keyword_len:]
-                        certified_item = certified_item_first
-                        continue
+                    end_pos = developer.find('\f-')
+                    if end_pos == -1:
+                        end_pos = developer.find('\fBSI')
+                    if end_pos == -1:
+                        end_pos = developer.find('Bundesamt')
+                    if end_pos != -1:
+                        developer = developer[:end_pos]
 
-                end_pos = developer.find('\f-')
-                if end_pos == -1:
-                    end_pos = developer.find('\fBSI')
-                if end_pos == -1:
-                    end_pos = developer.find('Bundesamt')
-                if end_pos != -1:
-                    developer = developer[:end_pos]
+                    items_found[TAG_CERT_ID] = normalize_match_string(cert_id)
+                    items_found[TAG_CERT_ITEM] = normalize_match_string(
+                        certified_item)
+                    items_found[TAG_DEVELOPER] = normalize_match_string(developer)
+                    items_found[TAG_CERT_LAB] = 'BSI'
 
-                items_found[TAG_CERT_ID] = normalize_match_string(cert_id)
-                items_found[TAG_CERT_ITEM] = normalize_match_string(
-                    certified_item)
-                items_found[TAG_DEVELOPER] = normalize_match_string(developer)
-                items_found[TAG_CERT_LAB] = 'BSI'
+            #
+            # Process page with more detailed certificate info
+            # PP Conformance, Functionality, Assurance
+            rules_certificate_third = [
+                'PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified',
+            ]
 
-        #
-        # Process page with more detailed certificate info
-        # PP Conformance, Functionality, Assurance
-        rules_certificate_third = [
-            'PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified',
-        ]
+            whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+                file_name)
 
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            file_name)
+            for rule in rules_certificate_third:
+                rule_and_sep = rule + REGEXEC_SEP
 
-        for rule in rules_certificate_third:
-            rule_and_sep = rule + REGEXEC_SEP
+                for m in re.finditer(rule_and_sep, whole_text):
+                    # check if previous rules had at least one match
+                    if not TAG_CERT_ID in items_found.keys():
+                        print('ERROR: front page not found for file: {}'.format(file_name))
 
-            for m in re.finditer(rule_and_sep, whole_text):
-                # check if previous rules had at least one match
-                if not TAG_CERT_ID in items_found.keys():
-                    print('ERROR: front page not found for file: {}'.format(file_name))
+                    match_groups = m.groups()
+                    ref_protection_profiles = match_groups[0]
+                    cc_version = match_groups[1]
+                    cc_security_level = match_groups[2]
 
-                match_groups = m.groups()
-                ref_protection_profiles = match_groups[0]
-                cc_version = match_groups[1]
-                cc_security_level = match_groups[2]
+                    items_found[TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
+                        ref_protection_profiles)
+                    items_found[TAG_CC_VERSION] = normalize_match_string(
+                        cc_version)
+                    items_found[TAG_CC_SECURITY_LEVEL] = normalize_match_string(
+                        cc_security_level)
 
-                items_found[TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
-                    ref_protection_profiles)
-                items_found[TAG_CC_VERSION] = normalize_match_string(
-                    cc_version)
-                items_found[TAG_CC_SECURITY_LEVEL] = normalize_match_string(
-                    cc_security_level)
+            if no_match_yet:
+                files_without_match.append(file_name)
 
-        if no_match_yet:
-            files_without_match.append(file_name)
+            progress.update(1)
 
     print('\n*** Certificates without detected preface:')
     for file_name in files_without_match:
@@ -572,97 +583,95 @@ def search_only_headers_anssi(walk_dir: Path):
     print('***ANSSI HEADER SEARCH***')
     items_found_all = {}
     files_without_match = []
-    for file_name in search_files(walk_dir):
-        if not os.path.isfile(file_name):
-            continue
-        file_ext = file_name[file_name.rfind('.'):]
-        if file_ext != '.txt':
-            continue
-        #        print('*** {} ***'.format(file_name))
 
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            file_name)
+    files_to_process = get_files_to_process(walk_dir, '.txt')
+    with tqdm(total=len(files_to_process)) as progress:
+        for file_name in files_to_process:
+            whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+                file_name)
 
-        # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs
-        pos = whole_text.find('')
-        if pos != -1:
-            pos = whole_text.find('', pos)
+            # for ANSII and DCSSI certificates, front page starts only on third page after 2 newpage signs
+            pos = whole_text.find('')
             if pos != -1:
-                whole_text = whole_text[pos:]
+                pos = whole_text.find('', pos)
+                if pos != -1:
+                    whole_text = whole_text[pos:]
 
-        no_match_yet = True
-        other_rule_already_match = False
-        other_rule = ''
-        rule_index = -1
-        for rule in rules_certificate_preface:
-            rule_index += 1
-            rule_and_sep = rule[1] + REGEXEC_SEP
+            no_match_yet = True
+            other_rule_already_match = False
+            other_rule = ''
+            rule_index = -1
+            for rule in rules_certificate_preface:
+                rule_index += 1
+                rule_and_sep = rule[1] + REGEXEC_SEP
 
-            for m in re.finditer(rule_and_sep, whole_text):
-                if no_match_yet:
-                    items_found_all[file_name] = {}
-                    items_found_all[file_name] = {}
-                    items_found = items_found_all[file_name]
-                    items_found[TAG_HEADER_MATCH_RULES] = []
-                    no_match_yet = False
+                for m in re.finditer(rule_and_sep, whole_text):
+                    if no_match_yet:
+                        items_found_all[file_name] = {}
+                        items_found_all[file_name] = {}
+                        items_found = items_found_all[file_name]
+                        items_found[TAG_HEADER_MATCH_RULES] = []
+                        no_match_yet = False
 
-                # insert rule if at least one match for it was found
-                if rule not in items_found[TAG_HEADER_MATCH_RULES]:
-                    items_found[TAG_HEADER_MATCH_RULES].append(rule[1])
+                    # insert rule if at least one match for it was found
+                    if rule not in items_found[TAG_HEADER_MATCH_RULES]:
+                        items_found[TAG_HEADER_MATCH_RULES].append(rule[1])
 
-                if not other_rule_already_match:
-                    other_rule_already_match = True
-                    other_rule = rule
-                else:
-                    print(
-                        'WARNING: multiple rules are matching same certification document: ' + file_name)
+                    if not other_rule_already_match:
+                        other_rule_already_match = True
+                        other_rule = rule
+                    else:
+                        print(
+                            'WARNING: multiple rules are matching same certification document: ' + file_name)
 
-                num_rules_hits[rule[1]] += 1  # add hit to this rule
+                    num_rules_hits[rule[1]] += 1  # add hit to this rule
 
-                match_groups = m.groups()
+                    match_groups = m.groups()
 
-                index_next_item = 0
+                    index_next_item = 0
 
-                items_found[TAG_CERT_ID] = normalize_match_string(
-                    match_groups[index_next_item])
-                index_next_item += 1
-
-                items_found[TAG_CERT_ITEM] = normalize_match_string(
-                    match_groups[index_next_item])
-                index_next_item += 1
+                    items_found[TAG_CERT_ID] = normalize_match_string(
+                        match_groups[index_next_item])
+                    index_next_item += 1
 
-                if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION:
-                    items_found[TAG_CERT_ITEM_VERSION] = ''
-                else:
-                    items_found[TAG_CERT_ITEM_VERSION] = normalize_match_string(
+                    items_found[TAG_CERT_ITEM] = normalize_match_string(
                         match_groups[index_next_item])
                     index_next_item += 1
 
-                if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES:
-                    items_found[TAG_REFERENCED_PROTECTION_PROFILES] = ''
-                else:
-                    items_found[TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
+                    if rule[0] == HEADER_TYPE.HEADER_MISSING_CERT_ITEM_VERSION:
+                        items_found[TAG_CERT_ITEM_VERSION] = ''
+                    else:
+                        items_found[TAG_CERT_ITEM_VERSION] = normalize_match_string(
+                            match_groups[index_next_item])
+                        index_next_item += 1
+
+                    if rule[0] == HEADER_TYPE.HEADER_MISSING_PROTECTION_PROFILES:
+                        items_found[TAG_REFERENCED_PROTECTION_PROFILES] = ''
+                    else:
+                        items_found[TAG_REFERENCED_PROTECTION_PROFILES] = normalize_match_string(
+                            match_groups[index_next_item])
+                        index_next_item += 1
+
+                    items_found[TAG_CC_VERSION] = normalize_match_string(
                         match_groups[index_next_item])
                     index_next_item += 1
 
-                items_found[TAG_CC_VERSION] = normalize_match_string(
-                    match_groups[index_next_item])
-                index_next_item += 1
+                    items_found[TAG_CC_SECURITY_LEVEL] = normalize_match_string(
+                        match_groups[index_next_item])
+                    index_next_item += 1
 
-                items_found[TAG_CC_SECURITY_LEVEL] = normalize_match_string(
-                    match_groups[index_next_item])
-                index_next_item += 1
+                    items_found[TAG_DEVELOPER] = normalize_match_string(
+                        match_groups[index_next_item])
+                    index_next_item += 1
 
-                items_found[TAG_DEVELOPER] = normalize_match_string(
-                    match_groups[index_next_item])
-                index_next_item += 1
+                    items_found[TAG_CERT_LAB] = normalize_match_string(
+                        match_groups[index_next_item])
+                    index_next_item += 1
 
-                items_found[TAG_CERT_LAB] = normalize_match_string(
-                    match_groups[index_next_item])
-                index_next_item += 1
+            if no_match_yet:
+                files_without_match.append(file_name)
 
-        if no_match_yet:
-            files_without_match.append(file_name)
+            progress.update(1)
 
     print('\n*** Certificates without detected preface:')
     for file_name in files_without_match:
@@ -758,291 +767,288 @@ def search_pp_only_headers(walk_dir: Path):
     print("***PP HEADER SEARCH***")
     items_found_all = {}
     files_without_match = []
-    for file_name in search_files(walk_dir):
-        if not os.path.isfile(file_name):
-            continue
-        file_ext = file_name[file_name.rfind('.'):]
-        if file_ext != '.txt':
-            continue
-        #        print('*** {} ***'.format(file_name))
+    files_to_process = get_files_to_process(walk_dir, '.txt')
+    with tqdm(total=len(files_to_process)) as progress:
+        for file_name in files_to_process:
+            #
+            # Process page with more detailed protection profile info
+            # PP Reference
 
-        #
-        # Process page with more detailed protection profile info
-        # PP Reference
+            whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
+                file_name)
 
-        whole_text, whole_text_with_newlines, was_unicode_decode_error = load_cert_file(
-            file_name)
+            no_match_yet = True
+            for rule in rules_pp_third:
+                rule_and_sep = rule[1] + REGEXEC_SEP
 
-        no_match_yet = True
-        for rule in rules_pp_third:
-            rule_and_sep = rule[1] + REGEXEC_SEP
+                for m in re.finditer(rule_and_sep, whole_text):
+                    if no_match_yet:
+                        items_found_all[file_name] = {}
+                        items_found_all[file_name] = {}
+                        items_found = items_found_all[file_name]
+                        items_found[TAG_HEADER_MATCH_RULES] = []
+                        no_match_yet = False
 
-            for m in re.finditer(rule_and_sep, whole_text):
-                if no_match_yet:
-                    items_found_all[file_name] = {}
-                    items_found_all[file_name] = {}
-                    items_found = items_found_all[file_name]
-                    items_found[TAG_HEADER_MATCH_RULES] = []
-                    no_match_yet = False
+                    # insert rule if at least one match for it was found
+                    if rule[1] not in items_found[TAG_HEADER_MATCH_RULES]:
+                        items_found[TAG_HEADER_MATCH_RULES].append(rule[1])
 
-                # insert rule if at least one match for it was found
-                if rule[1] not in items_found[TAG_HEADER_MATCH_RULES]:
-                    items_found[TAG_HEADER_MATCH_RULES].append(rule[1])
+                    match_groups = m.groups()
+                    index = 0
 
-                match_groups = m.groups()
-                index = 0
+                    if rule[0] == HEADER_TYPE.BSI_TYPE1:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_VERSION,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_GENERAL_STATUS,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_ID,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        keywords = match_groups[index].lstrip('  ')
+                        set_match_string(items_found, TAG_KEYWORDS, normalize_match_string(
+                            keywords[0:keywords.find('  ')]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_AUTHORS, 'BSI')
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'BSI')
 
-                if rule[0] == HEADER_TYPE.BSI_TYPE1:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_VERSION,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_GENERAL_STATUS,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_ID,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    keywords = match_groups[index].lstrip('  ')
-                    set_match_string(items_found, TAG_KEYWORDS, normalize_match_string(
-                        keywords[0:keywords.find('  ')]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_AUTHORS, 'BSI')
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'BSI')
+                    if rule[0] == HEADER_TYPE.BSI_TYPE2:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_DATE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_AUTHORS,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_REGISTRATOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_ID,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_VERSION,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        keywords = match_groups[index].lstrip('  ')
+                        set_match_string(items_found, TAG_KEYWORDS, normalize_match_string(
+                            keywords[0:keywords.find('  ')]))
+                        index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'BSI')
 
-                if rule[0] == HEADER_TYPE.BSI_TYPE2:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_DATE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_AUTHORS,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_REGISTRATOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_ID,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_VERSION,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    keywords = match_groups[index].lstrip('  ')
-                    set_match_string(items_found, TAG_KEYWORDS, normalize_match_string(
-                        keywords[0:keywords.find('  ')]))
-                    index += 1
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'BSI')
+                    if rule[0] == HEADER_TYPE.ANSSI_TYPE1:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_DATE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_REGISTRATOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_SPONSOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_EDITOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_REVIEWER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_VERSION,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        level = match_groups[index].lstrip('  ')
+                        set_match_string(items_found, TAG_CC_SECURITY_LEVEL, normalize_match_string(
+                            level[0:level.find('  ')]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.ANSSI_TYPE1:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_DATE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_REGISTRATOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_SPONSOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_EDITOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_REVIEWER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_VERSION,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    level = match_groups[index].lstrip('  ')
-                    set_match_string(items_found, TAG_CC_SECURITY_LEVEL, normalize_match_string(
-                        level[0:level.find('  ')]))
-                    index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI')
 
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI')
+                    if rule[0] == HEADER_TYPE.ANSSI_TYPE2:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_AUTHORS,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_REGISTRATOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_VERSION,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_KEYWORDS,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.ANSSI_TYPE2:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_AUTHORS,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_REGISTRATOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_VERSION,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_KEYWORDS,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI')
 
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI')
+                    if rule[0] == HEADER_TYPE.ANSSI_TYPE3:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        # todo: parse if multiple pp ids are present
+                        set_match_string(items_found, TAG_PP_ID,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_EDITOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_DATE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_SPONSOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        ccversion = match_groups[index].lstrip('  ')
+                        set_match_string(items_found, TAG_CC_VERSION, normalize_match_string(
+                            ccversion[0:ccversion.find('  ')]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.ANSSI_TYPE3:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    # todo: parse if multiple pp ids are present
-                    set_match_string(items_found, TAG_PP_ID,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_EDITOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_DATE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_SPONSOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    ccversion = match_groups[index].lstrip('  ')
-                    set_match_string(items_found, TAG_CC_VERSION, normalize_match_string(
-                        ccversion[0:ccversion.find('  ')]))
-                    index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI')
 
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'ANSSI')
+                    if rule[0] == HEADER_TYPE.DCSSI_TYPE1:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_ID,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_DATE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        author = match_groups[index].lstrip('  ')
+                        set_match_string(items_found, TAG_PP_AUTHORS, normalize_match_string(
+                            author[0:author.find('  ')]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.DCSSI_TYPE1:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_ID,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_DATE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    author = match_groups[index].lstrip('  ')
-                    set_match_string(items_found, TAG_PP_AUTHORS, normalize_match_string(
-                        author[0:author.find('  ')]))
-                    index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
 
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
+                    if rule[0] == HEADER_TYPE.DCSSI_TYPE2:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_AUTHORS,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        version = match_groups[index].lstrip('  ')
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER, normalize_match_string(
+                            version[0:version.find('  ')]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.DCSSI_TYPE2:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_AUTHORS,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    version = match_groups[index].lstrip('  ')
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER, normalize_match_string(
-                        version[0:version.find('  ')]))
-                    index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
 
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
+                    if rule[0] == HEADER_TYPE.FRONT_DCSSI_TYPE3 or rule[0] == HEADER_TYPE.FRONT_DCSSI_TYPE4:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_DATE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_ID,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_ID_REGISTRATOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.FRONT_DCSSI_TYPE3 or rule[0] == HEADER_TYPE.FRONT_DCSSI_TYPE4:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_DATE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_ID,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_ID_REGISTRATOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
 
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
+                    if rule[0] == HEADER_TYPE.DCSSI_TYPE5:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_AUTHORS,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_DATE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_SPONSOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        ccversion = match_groups[index].lstrip('  ')
+                        set_match_string(items_found, TAG_CC_VERSION, normalize_match_string(
+                            ccversion[0:ccversion.find('  ')]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.DCSSI_TYPE5:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_AUTHORS,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_DATE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_SPONSOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    ccversion = match_groups[index].lstrip('  ')
-                    set_match_string(items_found, TAG_CC_VERSION, normalize_match_string(
-                        ccversion[0:ccversion.find('  ')]))
-                    index += 1
+                    if rule[0] == HEADER_TYPE.DCSSI_TYPE6:
+                        set_match_string(items_found, TAG_PP_TITLE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_AUTHORS,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_VERSION_NUMBER,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_DATE,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_PP_SPONSOR,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_VERSION,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
+                        set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
+                                         normalize_match_string(match_groups[index]))
+                        index += 1
 
-                if rule[0] == HEADER_TYPE.DCSSI_TYPE6:
-                    set_match_string(items_found, TAG_PP_TITLE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_AUTHORS,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_VERSION_NUMBER,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_DATE,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_PP_SPONSOR,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_VERSION,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
-                    set_match_string(items_found, TAG_CC_SECURITY_LEVEL,
-                                     normalize_match_string(match_groups[index]))
-                    index += 1
+                        set_match_string(
+                            items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
 
-                    set_match_string(
-                        items_found, TAG_PP_REGISTRATOR_SIMPLIFIED, 'DCSSI')
+            if no_match_yet:
+                files_without_match.append(file_name)
 
-        if no_match_yet:
-            files_without_match.append(file_name)
+            progress.update(1)
 
     print('\n*** Protection profiles without detected header:')
     for file_name in files_without_match:
@@ -1064,36 +1070,105 @@ def extract_protectionprofiles_frontpage(walk_dir: Path):
     return pp_items_found
 
 
+def extract_keywords(params):
+    file_name, fragments_dir, file_prefix, should_censure_right_away, fips_items = params
+    result, modified_cert_file = parse_cert_file(
+        file_name, fips_rules if fips_items else rules, -1, LINE_SEPARATOR, should_censure_right_away, fips_items)
+
+    # save report text with highlighted/replaced matches into \\fragments\\ directory
+    save_fragments = True
+    if save_fragments:
+        base_path = file_name[:file_name.rfind(os.sep)]
+        file_name_short = file_name[file_name.rfind(os.sep) + 1:]
+        target_file = fragments_dir / file_name_short
+        save_modified_cert_file(
+            target_file, modified_cert_file[0], modified_cert_file[1])
+
+    return file_name, result
+
+
+def extract_certificates_keywords_parallel(walk_dir: Path, fragments_dir: Path, file_prefix, num_threads: int, should_censure_right_away=False, fips_items=None):
+    print("***EXTRACT KEYWORDS***")
+    all_items_found = {}
+
+    files_to_process = get_files_to_process(walk_dir, '.txt')
+    responses = []
+
+    with tqdm(total=len(files_to_process)) as progress:
+        with Pool(num_threads) as p:
+            batch_len = num_threads * 4
+            params = []
+            to_process = 0
+            for file_name in files_to_process:
+                to_process = to_process + 1
+                #params.append((file_name, fragments_dir, file_prefix, should_censure_right_away, fips_items, progress))
+                params.append((file_name, fragments_dir, file_prefix, should_censure_right_away, fips_items))
+
+                if len(params) == batch_len or to_process == len(files_to_process):
+                    results = p.map(extract_keywords, params)
+                    for response in results:
+                        file_name = response[0]
+                        fips_cert_name = os.path.splitext(
+                            os.path.splitext(os.path.basename(file_name))[0])[0]
+                        all_items_found[fips_cert_name if fips_items else file_name] = response[1]
+
+                    progress.update(batch_len)
+                    params = []
+
+    total_items_found = 0
+    for file_name in all_items_found:
+        total_items_found += count_num_items_found(all_items_found[file_name])
+
+    PRINT_MATCHES = True
+    if PRINT_MATCHES:
+        all_matches = []
+        for file_name in all_items_found:
+            print('*' * 10, "FILENAME:", file_name, '*' * 10)
+            for rule_group in all_items_found[file_name].keys():
+                items_found = all_items_found[file_name][rule_group]
+                for rule in items_found.keys():
+                    for match in items_found[rule]:
+                        if match not in all_matches:
+                            print(match)
+        #                            all_matches.append(match)
+
+        sorted_all_matches = sorted(all_matches)
+    #        for match in sorted_all_matches:
+    #            print(match)
+
+    # verify total matches found
+    print('\nTotal matches found: {}'.format(total_items_found))
+
+    return all_items_found
+
+
 def extract_certificates_keywords(walk_dir: Path, fragments_dir: Path, file_prefix, should_censure_right_away=False, fips_items=None):
     print("***EXTRACT KEYWORDS***")
     all_items_found = {}
     # cert_id = {}
-    for file_name in search_files(walk_dir):
-        if not os.path.isfile(file_name):
-            continue
-        file_ext = file_name[file_name.rfind('.'):]
-        if file_ext != '.txt':
-            continue
 
-        #        print('*** {} ***'.format(file_name))
+    files_to_process = get_files_to_process(walk_dir, '.txt')
+    with tqdm(total=len(files_to_process)) as progress:
+        for file_name in files_to_process:
+            fips_cert_name = os.path.splitext(
+                os.path.splitext(os.path.basename(file_name))[0])[0]
+            # parse certificate, return all matches
+            all_items_found[fips_cert_name if fips_items else file_name], modified_cert_file = parse_cert_file(
+                file_name, fips_rules if fips_items else rules, -1, LINE_SEPARATOR, should_censure_right_away=should_censure_right_away,
+                fips_items=fips_items)
 
-        fips_cert_name = os.path.splitext(
-            os.path.splitext(os.path.basename(file_name))[0])[0]
-        # parse certificate, return all matches
-        all_items_found[fips_cert_name if fips_items else file_name], modified_cert_file = parse_cert_file(
-            file_name, fips_rules if fips_items else rules, -1, should_censure_right_away=should_censure_right_away,
-            fips_items=fips_items)
+            # try to establish the certificate id of the current certificate
+            # cert_id[file_cert_name] = estimate_cert_id(
+            #     None, all_items_found[file_cert_name], file_name)
 
-        # try to establish the certificate id of the current certificate
-        # cert_id[file_cert_name] = estimate_cert_id(
-        #     None, all_items_found[file_cert_name], file_name)
+            # save report text with highlighted/replaced matches into \\fragments\\ directory
+            base_path = file_name[:file_name.rfind(os.sep)]
+            file_name_short = file_name[file_name.rfind(os.sep) + 1:]
+            target_file = fragments_dir / file_name_short
+            save_modified_cert_file(
+                target_file, modified_cert_file[0], modified_cert_file[1])
 
-        # save report text with highlighted/replaced matches into \\fragments\\ directory
-        base_path = file_name[:file_name.rfind(os.sep)]
-        file_name_short = file_name[file_name.rfind(os.sep) + 1:]
-        target_file = fragments_dir / file_name_short
-        save_modified_cert_file(
-            target_file, modified_cert_file[0], modified_cert_file[1])
+            progress.update(1)
 
     # print('\nTotal matches found in separate files:')
     # print_total_matches_in_files(all_items_found_count)
@@ -1130,51 +1205,112 @@ def extract_certificates_keywords(walk_dir: Path, fragments_dir: Path, file_pref
     return all_items_found
 
 
+def extract_pdf(params):
+    file_name = params
+
+    item = {}
+    item['pdf_file_size_bytes'] = os.path.getsize(file_name)
+    try:
+        with open(file_name, 'rb') as f:
+            pdf = PdfFileReader(f)
+            # store additional interesting info
+            item['pdf_is_encrypted'] = pdf.getIsEncrypted()
+            item['pdf_number_of_pages'] = pdf.getNumPages()
+
+            # extract pdf metadata (as dict) and save it
+            info = pdf.getDocumentInfo()
+            if info is not None:
+                for key in info:
+                    item[key] = str(info[key])
+    except Exception as e:
+        item['error'] = str(e)
+
+    return file_name, item
+
+
+def extract_certificates_pdfmeta_parallel(walk_dir: Path, file_prefix, num_threads: int):
+    all_items_found = {}
+    counter = 0
+
+    print("***EXTRACT PDFMETA***")
+    files_to_process = get_files_to_process(walk_dir, '.pdf')
+    with tqdm(total=len(files_to_process)) as progress:
+        with Pool(num_threads) as p:
+            batch_len = num_threads * 4
+            params = []
+            to_process = 0
+            for file_name in files_to_process:
+                to_process = to_process + 1
+
+                params.append((file_name))
+
+                if len(params) == batch_len or to_process == len(files_to_process):
+                    results = p.map(extract_pdf, params)
+                    for response in results:
+                        file_name = response[0]
+                        all_items_found[file_name] = response[1]
+
+                    progress.update(batch_len)
+                    params = []
+
+                write_intermediate = False
+                if write_intermediate:
+                    if counter % 100 == 0:
+                        # store results into file with fixed name
+                        with open("{}_data_pdfmeta_{}.json".format(file_prefix, counter), "w",
+                                  errors=FILE_ERRORS_STRATEGY) as write_file:
+                            json.dump(all_items_found, write_file, indent=4, sort_keys=True)
+                counter += 1
+
+    return all_items_found
+
+
 def extract_certificates_pdfmeta(walk_dir: Path, file_prefix, results_dir: Path):
     all_items_found = {}
     counter = 0
-    for file_name in search_files(walk_dir):
-        if not os.path.isfile(file_name):
-            continue
-        file_ext = file_name[file_name.rfind('.'):]
-        if file_ext != '.pdf':
-            continue
 
-        print("***EXTRACT PDFMETA***")
-        #        print('*** {} ***'.format(file_name))
+    print("***EXTRACT PDFMETA***")
+    files_to_process = get_files_to_process(walk_dir, '.pdf')
+    with tqdm(total=len(files_to_process)) as progress:
+        for file_name in files_to_process:
+            #        print('*** {} ***'.format(file_name))
 
-        item = {}
-        item['pdf_file_size_bytes'] = os.path.getsize(file_name)
-        try:
-            with open(file_name, 'rb') as f:
-                pdf = PdfFileReader(f)
-                # store additional interesting info
-                item['pdf_is_encrypted'] = pdf.getIsEncrypted()
-                item['pdf_number_of_pages'] = pdf.getNumPages()
+            item = {}
+            item['pdf_file_size_bytes'] = os.path.getsize(file_name)
+            try:
+                with open(file_name, 'rb') as f:
+                    pdf = PdfFileReader(f)
+                    # store additional interesting info
+                    item['pdf_is_encrypted'] = pdf.getIsEncrypted()
+                    item['pdf_number_of_pages'] = pdf.getNumPages()
 
-                # extract pdf metadata (as dict) and save it
-                info = pdf.getDocumentInfo()
-                if info is not None:
-                    for key in info:
-                        item[key] = str(info[key])
-        except Exception as e:
-            item['error'] = str(e)
+                    # extract pdf metadata (as dict) and save it
+                    info = pdf.getDocumentInfo()
+                    if info is not None:
+                        for key in info:
+                            item[key] = str(info[key])
+            except Exception as e:
+                item['error'] = str(e)
 
-        # test save of the data extracted to prevent error only very later
-        # try:
-        #     with open("{}_temp.json".format(file_prefix), "w") as write_file:
-        #         write_file.write(json.dumps(item, indent=4, sort_keys=True))
-        # except Exception:
-        #     print('  ERROR: invalid data from pdf')
+            # test save of the data extracted to prevent error only very later
+            # try:
+            #     with open("{}_temp.json".format(file_prefix), "w") as write_file:
+            #         write_file.write(json.dumps(item, indent=4, sort_keys=True))
+            # except Exception:
+            #     print('  ERROR: invalid data from pdf')
 
-        all_items_found[file_name] = item
+            all_items_found[file_name] = item
 
-        if counter % 100 == 0:
-            # store results into file with fixed name
-            with open("{}_data_pdfmeta_{}.json".format(file_prefix, counter), "w",
-                      errors=FILE_ERRORS_STRATEGY) as write_file:
-                json.dump(all_items_found, write_file, indent=4, sort_keys=True)
-        counter += 1
+            write_intermediate = False
+            if write_intermediate:
+                if counter % 100 == 0:
+                    # store results into file with fixed name
+                    with open("{}_data_pdfmeta_{}.json".format(file_prefix, counter), "w",
+                              errors=FILE_ERRORS_STRATEGY) as write_file:
+                        json.dump(all_items_found, write_file, indent=4, sort_keys=True)
+            counter += 1
+
+            progress.update(1)
 
     return all_items_found