diff options
| -rw-r--r-- | examples/fips_oop_demo.py | 40 | ||||
| -rw-r--r-- | sec_certs/cert_rules.py | 107 | ||||
| -rw-r--r-- | sec_certs/certificate/fips.py | 653 | ||||
| -rw-r--r-- | sec_certs/constants.py | 2 | ||||
| -rw-r--r-- | sec_certs/helpers.py | 18 | ||||
| -rw-r--r-- | sec_certs/settings.yaml | 11 | ||||
| -rw-r--r-- | tests/data/test_fips_oop/algorithms.json | 513 | ||||
| -rw-r--r-- | tests/fips_test_utils.py | 51 | ||||
| -rw-r--r-- | tests/settings_test.yaml | 19 | ||||
| -rw-r--r-- | tests/test_fips_oop.py | 134 |
10 files changed, 1421 insertions, 127 deletions
diff --git a/examples/fips_oop_demo.py b/examples/fips_oop_demo.py index dde76101..d27429fc 100644 --- a/examples/fips_oop_demo.py +++ b/examples/fips_oop_demo.py @@ -5,69 +5,61 @@ import click from sec_certs.dataset.fips import FIPSDataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset from sec_certs.configuration import config -from sec_certs.helpers import analyze_matched_algs + @click.command() @click.option('--config-file', help='Path to config file') @click.option('--json-file', help='Path to dataset json file') -@click.option('--no-download-algs', help='don\'t download algs', is_flag=True) -def main(config_file, json_file, no_download_algs): +@click.option('--no-download-algs', help='Redo scan of html files', is_flag=True) +@click.option('--redo-web-scan', help='Redo scan of PDF files', is_flag=True) +@click.option('--redo-keyword-scan', help='Don\'t download algs', is_flag=True) +@click.option('--higher-precision-results', + help='Redo table search for certificates with high error rate. Behaviour undefined if used on a newly instantiated dataset.', + is_flag=True) +def main(config_file, json_file, no_download_algs, redo_web_scan, redo_keyword_scan, higher_precision_results): logging.basicConfig(level=logging.INFO) start = datetime.now() # Load config - config.load(config_file if config_file else 'sec_certs/settings.yaml') + config.load(config_file if config_file else '../sec_certs/settings.yaml') # Create empty dataset - dset = FIPSDataset({}, Path('./fips_dataset'), 'sample_dataset', 'sample dataset description') + dset = FIPSDataset({}, Path('../fips_dataset'), 'sample_dataset', 'sample dataset description') # this is for creating test dataset, usually with small number of pdfs # dset = FIPSDataset({}, Path('./fips_test_dataset'), 'small dataset', 'small dataset for keyword testing') # Load metadata for certificates from CSV and HTML sources - dset.get_certs_from_web(json_file=json_file, redo=True) + dset.get_certs_from_web(json_file=json_file, redo=redo_web_scan) logging.info(f'Finished parsing. Have dataset with {len(dset)} certificates.') - # Dump dataset into JSON - dset.to_json(dset.root_dir / 'fips_full_dataset.json') logging.info(f'Dataset saved to {dset.root_dir}/fips_full_dataset.json') logging.info("Converting pdfs") dset.convert_all_pdfs() - dset.to_json(dset.root_dir / 'fips_full_dataset.json') logging.info("Extracting keywords now.") - dset.extract_keywords(redo=True) + dset.extract_keywords(redo=redo_keyword_scan) logging.info(f'Finished extracting certificates for {len(dset.certs)} items.') - logging.info("Dumping dataset again...") - dset.to_json(dset.root_dir / 'fips_full_dataset.json') logging.info("Searching for tables in pdfs") - not_decoded_files = dset.extract_certs_from_tables() + not_decoded_files = dset.extract_certs_from_tables(higher_precision_results) logging.info(f"Done. Files not decoded: {not_decoded_files}") - dset.to_json(dset.root_dir / 'fips_mentioned.json') logging.info("Parsing algorithms") if not no_download_algs: - aset = FIPSAlgorithmDataset({}, Path('fips_dataset/web/algorithms'), 'algorithms', 'sample algs') + aset = FIPSAlgorithmDataset({}, Path(dset.root_dir / 'web/algorithms'), 'algorithms', 'sample algs') aset.get_certs_from_web() + logging.info(f'Finished parsing. Have algorithm dataset with {len(aset)} algorithm numbers.') dset.algorithms = aset logging.info("finalizing results.") - dset.finalize_results() - logging.info('dump again') - dset.to_json(dset.root_dir / 'fips_full_dataset.json') - - dset.get_dot_graph('different_new') - - data = dset.match_algs() - analyze_matched_algs(data) - + dset.plot_graphs(show=False) end = datetime.now() logging.info(f'The computation took {(end - start)} seconds.') diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py index 3be285c5..ba857300 100644 --- a/sec_certs/cert_rules.py +++ b/sec_certs/cert_rules.py @@ -321,64 +321,65 @@ rules_other = [ rules_fips_remove_algorithm_ids = [ # --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) --- - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?\s?(\d{4})", - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?\s?(\d{3})", - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?\s?(\d{2})", - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?\s?(\d{1})", +# + added (and #id) everywhere + r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*and[\s#]*\d+)?", + r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*and[\s#]*\d+)?", + r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*and[\s#]*\d+)?", + r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*and[\s#]*\d+)?", # --- same as above, without hw or fw --- - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", + r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", + r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", + r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", + r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", # --- SHS/A - (bits) (method) ((cert #) numbers) --- - r"SH[SA][-– ]*(?:160|224|256|384|512)?(?:[\s(\[]*?(?:KAT)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)(\d{4})(?:\)?\[#?\d+\])?", - r"SH[SA][-– ]*(?:160|224|256|384|512)?(?:[\s(\[]*?(?:KAT)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)(\d{3})(?:\)?\[#?\d+\])?", - r"SH[SA][-– ]*(?:160|224|256|384|512)?(?:[\s(\[]*?(?:KAT)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)(\d{2})(?:\)?\[#?\d+\])?", - r"SH[SA][-– ]*(?:160|224|256|384|512)?(?:[\s(\[]*?(?:KAT)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)(\d{1})(?:\)?\[#?\d+\])?", + r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", + r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", + r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", + r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?", # --- RSA (bits) (method) ((cert #)) --- - r"RSA(?:[-– ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"RSA(?:[-– ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"RSA(?:[-– ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"RSA(?:[-– ]*(?:512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", + r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", + r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", + r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", + r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", # --- RSA (SSA) (PKCS) (version) (#) --- - r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{4})?", - r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{3})?", - r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{2})?", - r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?\s?(\d{1})?", + r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{4})?", + r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{3})?", + r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{2})?", + r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{1})?", # --- AES (bits) (method) ((cert #)) --- - r"AES[-– ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})(?:\)?\[#?\d+\])?", - r"AES[-– ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})(?:\)?\[#?\d+\])?", - r"AES[-– ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})(?:\)?\[#?\d+\])?", - r"AES[-– ]*((?:128|192|256|)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})(?:\)?\[#?\d+\])?", + r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", + r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", + r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", + r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?", # --- Diffie Helman (CVL) ((cert #)) --- - r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?\s?(\d{4})", - r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?\s?(\d{3})", - r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?\s?(\d{2})", - r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?\s?(\d{1})", + r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{4})", + r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{3})", + r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{2})", + r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{1})", # --- DRBG (bits) (method) (cert #) --- - r"DRBG[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"DRBG[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"DRBG[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"DRBG[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", + r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", + r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", + r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", + r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", # --- DES (bits) (method) (cert #) - r"DES[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"DES[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"DES[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"DES[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", + r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*?and[\s#]*?(\d+))?", + r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*?and[\s#]*?(\d+))?", + r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*?and[\s#]*?(\d+))?", + r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*?and[\s#]*?(\d+))?", # --- DSA (bits) (method) (cert #) - r"DSA[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{4})", - r"DSA[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{3})", - r"DSA[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{2})", - r"DSA[ –-]*((?:160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?\s?(\d{1})", + r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})", + r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})", + r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})", + r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})", # --- platforms (#)+ - this is used in modification history --- r"[Pp]latforms? #\d+(?:#\d+|,| |-|and)*[^\n]*", @@ -400,12 +401,14 @@ rules_fips_remove_algorithm_ids = [ # --- PKCS (#) --- r"PKCS[\s]?#?\d+", - r"PKSC[\s]?#?\d+" # typo, #625 -] + r"PKSC[\s]?#?\d+", # typo, #625 +# --- # C and # A (just in case) --- + r"#\s+?[Cc]\d+", + r"#\s+?[Aa]\d+" +] rules_fips_to_remove = [ # --- random words found --- - r"Survey #192", # why would they get an address like this /o\ cert 2079 r"[Ss]lot #\d", # a card slot, #2069 r"[Ss]eals? ?\(?#\d - #\d", # #1232 r"\[#\d*\]", # some certs use this as references @@ -413,16 +416,19 @@ rules_fips_to_remove = [ r"[Pp]ower [Ss]upply #\d", # #604 r"TEL #\d and #\d", # #3337 r"#\d+ - #\d+", # labels, seals... #1232 + r"#\d+‐#?\d+", # labels, seals... #3530 + r"#\d+ to #?\d+", # labels, seals... #3058 + r"see #\d+", # labels, seals... #3058 r"#\d+, ?#\d+", r"#?\d+ and #?\d+", r"label \(#\d+\)", r"[Ll]abel #\d+", r"\(#\d\)", r"IETF[25\s]*RFC[26\s]*#\d+", # #3425 - r"Bendix Road North #760", # #3325 - r"5080 Spectrum Drive, #1000E", r"Document # 540-105000-A1", r"Certificate #2287-1 from EMCE Engineering", # ??? + r"[sS]cenarios?\s?#\d+", # 3789 + r"#\d+\s?\(\S\)", # 2159 ] rules_fips_cert = [ @@ -430,10 +436,10 @@ rules_fips_cert = [ # r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{3})", # r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{2})", # r"(?:#\s?|Cert\.?[^. ]*?\s?)(?P<id>\d{1}) - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{4}[^\d])", - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{3}[^\d])", - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2}[^\d])", - r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1}[^\d])" + r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{4})(?!\d)", + r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{3})(?!\d)", + r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{2})(?!\d)", + r"(?:#[^\S\r\n]?|Cert\.?(?!.\s)[^\S\r\n]?|Certificate[^\S\r\n]?)(?P<id>\d{1})(?!\d)" ] # rule still too "general" @@ -508,8 +514,7 @@ fips_rules['rules_to_remove'] = rules_fips_to_remove fips_rules['rules_security_level'] = rules_fips_security_level fips_rules['rules_cert_id'] = rules_fips_cert fips_common_rules = copy.deepcopy(common_rules) # make separate copy not to process cc rules by fips's re.compile -#fips_rules.update(fips_common_rules) for rule in fips_rules: for current_rule in range(len(fips_rules[rule])): - fips_rules[rule][current_rule] = re.compile(fips_rules[rule][current_rule])
\ No newline at end of file + fips_rules[rule][current_rule] = re.compile(fips_rules[rule][current_rule]) diff --git a/sec_certs/certificate/fips.py b/sec_certs/certificate/fips.py index 5819cbc4..8d8d80fc 100644 --- a/sec_certs/certificate/fips.py +++ b/sec_certs/certificate/fips.py @@ -86,7 +86,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): algorithms: Optional[List[Dict[str, str]]] tested_conf: Optional[List[str]] description: Optional[str] - mentioned_certs: Optional[List[str]] + mentioned_certs: Optional[Dict[str, Dict[str, int]]] vendor: Optional[str] vendor_www: Optional[str] lab: Optional[str] @@ -100,6 +100,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): revoked_link: Optional[str] sw_versions: Optional[str] product_url: Optional[str] + connections: List[str] def __post_init__(self): self.date_validation = [parser.parse(x).date() for x in @@ -131,6 +132,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): cert_id: int keywords: Dict algorithms: List + connections: List[str] @property def dgst(self): @@ -153,9 +155,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): @dataclass(eq=True) class Processed(ComplexSerializableType): - keywords: Optional[Dict] - algorithms: Dict - connections: List + keywords: Optional[Dict[str, Dict]] + algorithms: Dict[str, Dict] + connections: List[str] + unmatched_algs: int @property def dgst(self): @@ -219,26 +222,27 @@ class FIPSCertificate(Certificate, ComplexSerializableType): 'type': None, 'embodiment': None, 'tested_conf': None, 'description': None, 'vendor': None, 'vendor_www': None, 'lab': None, 'lab_nvlap': None, 'historical_reason': None, 'revoked_reason': None, 'revoked_link': None, 'algorithms': [], - 'mentioned_certs': [], 'tables_done': False, 'security_policy_www': None, 'certificate_www': None, + 'mentioned_certs': {}, 'tables_done': False, 'security_policy_www': None, 'certificate_www': None, 'hw_versions': None, 'fw_versions': None, 'sw_versions': None, 'product_url': None} return d @staticmethod - def parse_caveat(current_text: str) -> List: + def parse_caveat(current_text: str) -> Dict[str, Dict[str, int]]: """ Parses content of "Caveat" of FIPS CMVP .html file :param current_text: text of "Caveat" - :return: list of all found algorithm IDs + :return: dictionary of all found algorithm IDs """ - ids_found = [] - r_key = r"(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)(?P<id>\d+)" + ids_found = {} + r_key = r"(?P<word>\w+)?\s?(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)+(?P<id>\d+)" for m in re.finditer(r_key, current_text): - if r_key in ids_found and m.group() in ids_found[0]: - ids_found[0][m.group()]['count'] += 1 + if m.group('word') and m.group('word').lower() in {'rsa', 'shs', 'dsa', 'pkcs', 'aes'}: + continue + if m.group('id') in ids_found: + ids_found[m.group('id')]['count'] += 1 else: - ids_found.append( - {r"(?:#\s?|Cert\.?(?!.\s)\s?|Certificate\s?)(?P<id>\d+?})": {m.group(): {'count': 1}}}) + ids_found[m.group('id')] = {'count': 1} return ids_found @@ -258,7 +262,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): for m in re.finditer(reg, current_text): set_items.add(m.group()) - return [{"Certificate": list(set_items)}] + return [{"Certificate": list(set_items)}] if len(set_items) > 0 else [] @staticmethod def parse_table(element: Union[Tag, NavigableString]) -> List[Dict]: @@ -271,9 +275,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): trs = element.find_all('tr') for tr in trs: tds = tr.find_all('td') + cert = FIPSCertificate.extract_algorithm_certificates(tds[1].text) found_items.append( {'Name': tds[0].text, - 'Certificate': FIPSCertificate.extract_algorithm_certificates(tds[1].text)[0]['Certificate'], + 'Certificate': cert[0]['Certificate'] if cert != [] else [], 'Links': [str(x) for x in tds[1].find_all('a')], 'Raw': str(tr)}) @@ -292,8 +297,8 @@ class FIPSCertificate(Certificate, ComplexSerializableType): elif 'caveat' in pairs[title]: html_items_found[pairs[title]] = content - html_items_found['mentioned_certs'] += FIPSCertificate.parse_caveat( - content) + html_items_found['mentioned_certs'].update(FIPSCertificate.parse_caveat( + content)) elif 'FIPS Algorithms' in title: html_items_found['algorithms'] += FIPSCertificate.parse_table( @@ -447,15 +452,18 @@ class FIPSCertificate(Certificate, ComplexSerializableType): items_found['revoked_reason'] if 'revoked_reason' in items_found else None, items_found['revoked_link'] if 'revoked_link' in items_found else None, items_found['sw_versions'] if 'sw_versions' in items_found else None, - items_found['product_url']) if 'product_url' in items_found else None, + items_found['product_url'] if 'product_url' in items_found else None, + [] + ), # connections FIPSCertificate.PdfScan( items_found['cert_id'], {} if not initialized else initialized.pdf_scan.keywords, - [] if not initialized else initialized.pdf_scan.algorithms - ), - FIPSCertificate.Processed(None, {}, []), - state - ) + [] if not initialized else initialized.pdf_scan.algorithms, + [] # connections + ), + FIPSCertificate.Processed(None, {}, [], 0), + state + ) @staticmethod def convert_pdf_file(tup: Tuple['FIPSCertificate', Path, Path]) -> 'FIPSCertificate': @@ -470,6 +478,17 @@ class FIPSCertificate(Certificate, ComplexSerializableType): cert.state.txt_state = True return cert + + @staticmethod + def _declare_state(text: str): + """ + If less then half of the text is formed of alphabet characters, + we declare the security policy as "non-parsable" + :param text: security policy content + :return: True if parsable, otherwise False + """ + return len(text) * 0.5 <= len(''.join(filter(str.isalpha, text))) + @staticmethod def find_keywords(cert: 'FIPSCertificate') -> Tuple[Optional[Dict], 'FIPSCertificate']: if not cert.state.txt_state: @@ -480,8 +499,12 @@ class FIPSCertificate(Certificate, ComplexSerializableType): text_to_parse = text_with_newlines if config.use_text_with_newlines_during_parsing else text - items_found, fips_text = FIPSCertificate.parse_cert_file(FIPSCertificate.remove_platforms(text_to_parse), - cert.web_scan.algorithms) + cert.state.txt_state = FIPSCertificate._declare_state(text) + + if config.ignore_first_page: + text_to_parse = text_to_parse[text_to_parse.index(" "):] + + items_found, fips_text = FIPSCertificate.parse_cert_file(FIPSCertificate.remove_platforms(text_to_parse)) save_modified_cert_file(cert.state.fragment_path.with_suffix( '.fips.txt'), fips_text, unicode_error) @@ -516,16 +539,11 @@ class FIPSCertificate(Certificate, ComplexSerializableType): for web_alg in alg_list: if ''.join(filter(str.isdigit, web_alg)) not in all_algorithms: not_found.append(web_alg) - logger.error( - f"For cert {cert.dgst}:\n\tNOT FOUND: {len(not_found)}\n" - f"\tFOUND: {sum([len(a['Certificate']) for a in cert.web_scan.algorithms]) - len(not_found)}") - logger.error(f"Not found: {not_found}") return len(not_found) @staticmethod def remove_platforms(text_to_parse: str): - pat = re.compile( - r"(?:modification|revision|change) history\n[\s\S]*? ", re.IGNORECASE) + pat = re.compile(r"(?:(?:modification|revision|change) history|version control)\n[\s\S]*? ", re.IGNORECASE) for match in pat.finditer(text_to_parse): text_to_parse = text_to_parse.replace( match.group(), 'x' * len(match.group())) @@ -533,7 +551,7 @@ class FIPSCertificate(Certificate, ComplexSerializableType): @staticmethod def parse_cert_file_common(text_to_parse: str, whole_text_with_newlines: str, - search_rules: Dict) -> Tuple[Optional[Dict], str]: + search_rules: Dict) -> Tuple[Optional[Dict[Pattern, Dict]], str]: # apply all rules items_found_all = {} for rule_group in search_rules.keys(): @@ -601,10 +619,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): return items_found_all, whole_text_with_newlines @staticmethod - def parse_cert_file(text_to_parse: str, algorithms: List[Dict]) \ - -> Tuple[Optional[Dict], str]: + def parse_cert_file(text_to_parse: str) -> Tuple[Optional[Dict[Pattern, Dict]], str]: # apply all rules items_found_all: Dict = {} + for rule_group in fips_rules.keys(): if rule_group not in items_found_all: items_found_all[rule_group] = {} @@ -636,21 +654,27 @@ class FIPSCertificate(Certificate, ComplexSerializableType): return items_found_all, text_to_parse @staticmethod - def analyze_tables(cert: 'FIPSCertificate') -> Tuple[bool, 'FIPSCertificate', List]: + def analyze_tables(tup: Tuple['FIPSCertificate', bool]) -> Tuple[bool, 'FIPSCertificate', List]: + cert, precision = tup + if not (precision and cert.state.tables_done) \ + or (precision and cert.processed.unmatched_algs < config.cert_threshold['value']): + return cert.state.tables_done, cert, [] + cert_file = cert.state.sp_path txt_file = cert_file.with_suffix('.pdf.txt') with open(txt_file, 'r', encoding='utf-8') as f: tables = helpers.find_tables(f.read(), txt_file) + all_pages = precision and cert.processed.unmatched_algs > config.cert_threshold['value'] # bool value lst: List = [] if tables: try: - data = read_pdf(cert_file, pages=tables, silent=True) + data = read_pdf(cert_file, pages='all' if all_pages else tables, silent=True) except Exception as e: try: logger.error(e) helpers.repair_pdf(cert_file) - data = read_pdf(cert_file, pages=tables, silent=True) + data = read_pdf(cert_file, pages='all' if all_pages else tables, silent=True) except Exception as ex: logger.error(ex) @@ -660,12 +684,12 @@ class FIPSCertificate(Certificate, ComplexSerializableType): for df in data: for col in range(len(df.columns)): if 'cert' in df.columns[col].lower() or 'algo' in df.columns[col].lower(): - lst += FIPSCertificate.extract_algorithm_certificates( + tmp = FIPSCertificate.extract_algorithm_certificates( df.iloc[:, col].to_string(index=False), True) - + lst += tmp if tmp != [{"Certificate": []}] else [] # Parse again if someone picks not so descriptive column names - lst += FIPSCertificate.extract_algorithm_certificates( - df.to_string(index=False)) + tmp = FIPSCertificate.extract_algorithm_certificates(df.to_string(index=False)) + lst += tmp if tmp != [{"Certificate": []}] else [] return True, cert, lst def _create_alg_set(self) -> Set: @@ -680,9 +704,10 @@ class FIPSCertificate(Certificate, ComplexSerializableType): return self.processed.keywords = copy.deepcopy(self.pdf_scan.keywords) + # TODO figure out why can't I delete this if self.web_scan.mentioned_certs: - for item in self.web_scan.mentioned_certs: - self.processed.keywords['rules_cert_id'].update(item) + for item, value in self.web_scan.mentioned_certs.items(): + self.processed.keywords['rules_cert_id'].update({'caveat_item': {item: value}}) alg_set = self._create_alg_set() @@ -713,5 +738,543 @@ class FIPSCertificate(Certificate, ComplexSerializableType): @staticmethod def get_compare(vendor: str): vendor_split = vendor.replace(',', '') \ - .replace('-', ' ').replace('+', ' ').replace('®', '').split() - return vendor_split[0] if len(vendor_split) > 0 else vendor + .replace('-', ' ').replace('+', ' ').replace('®', '').replace('(R)', '').split() + return vendor_split[0][:4] if len(vendor_split) > 0 else vendor + + +class CommonCriteriaCert(Certificate, ComplexSerializableType): + cc_url = 'http://www.commoncriteriaportal.org' + empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/' + + @dataclass(eq=True, frozen=True) + class MaintainanceReport(ComplexSerializableType): + """ + Object for holding maintainance reports. + """ + maintainance_date: date + maintainance_title: str + maintainance_report_link: str + maintainance_st_link: str + + def __post_init__(self): + super().__setattr__('maintainance_report_link', + helpers.sanitize_link(self.maintainance_report_link)) + super().__setattr__('maintainance_st_link', + helpers.sanitize_link(self.maintainance_st_link)) + super().__setattr__('maintainance_title', + helpers.sanitize_string(self.maintainance_title)) + super().__setattr__('maintainance_date', helpers.sanitize_date(self.maintainance_date)) + + def to_dict(self): + return copy.deepcopy(self.__dict__) + + @classmethod + def from_dict(cls, dct): + return cls(*tuple(dct.values())) + + def __lt__(self, other): + return self.maintainance_date < other.maintainance_date + + @dataclass(eq=True, frozen=True) + class ProtectionProfile(ComplexSerializableType): + """ + Object for holding protection profiles. + """ + pp_name: str + pp_link: Optional[str] + + def __post_init__(self): + super().__setattr__('pp_name', helpers.sanitize_string(self.pp_name)) + super().__setattr__('pp_link', helpers.sanitize_link(self.pp_link)) + + def to_dict(self): + return copy.deepcopy(self.__dict__) + + @classmethod + def from_dict(cls, dct): + return cls(*tuple(dct.values())) + + def __lt__(self, other): + return self.pp_name < other.pp_name + + @dataclass(init=False) + class InternalState(ComplexSerializableType): + st_link_ok: bool + report_link_ok: bool + st_convert_ok: bool + report_convert_ok: bool + st_extract_ok: bool + report_extract_ok: bool + st_pdf_path: Path + report_pdf_path: Path + st_txt_path: Path + report_txt_path: Path + errors: Optional[List[str]] + + def __init__(self, st_link_ok: bool = True, report_link_ok: bool = True, + st_convert_ok: bool = True, report_convert_ok: bool = True, + st_extract_ok: bool = True, report_extract_ok: bool = True, + errors: Optional[List[str]] = None): + self.st_link_ok = st_link_ok + self.report_link_ok = report_link_ok + self.st_convert_ok = st_convert_ok + self.report_convert_ok = report_convert_ok + self.st_extract_ok = st_extract_ok + self.report_extract_ok = report_extract_ok + + if errors is None: + self.errors = [] + else: + self.errors = errors + + def to_dict(self): + return {'st_link_ok': self.st_link_ok, 'report_link_ok': self.report_link_ok, + 'st_convert_ok': self.st_convert_ok, 'report_convert_ok': self.report_convert_ok, + 'st_extract_ok': self.st_extract_ok, 'report_extract_ok': self.report_extract_ok, + 'errors': self.errors} + + @classmethod + def from_dict(cls, dct: Dict[str, bool]): + return cls(*tuple(dct.values())) + + @dataclass(init=False) + class PdfData(ComplexSerializableType): + report_metadata: Dict[str, str] + st_metadata: Dict[str, str] + report_frontpage: Dict[str, str] + st_frontpage: Dict[str, str] + report_keywords: Dict[str, str] + st_keywords: Dict[str, str] + + def __init__(self, report_metadata: Optional[Dict[str, str]] = None, + st_metadata: Optional[Dict[str, str]] = None, + report_frontpage: Optional[Dict[str, str]] = None, st_frontpage: Optional[Dict[str, str]] = None, + report_keywords: Optional[Dict[str, str]] = None, st_keywords: Optional[Dict[str, str]] = None): + self.report_metadata = report_metadata + self.st_metadata = st_metadata + self.report_frontpage = report_frontpage + self.st_frontpage = st_frontpage + self.report_keywords = report_keywords + self.st_keywords = st_keywords + + def to_dict(self): + return {'report_metadata': self.report_metadata, 'st_metadata': self.st_metadata, + 'report_frontpage': self.report_frontpage, + 'st_frontpage': self.st_frontpage, 'report_keywords': self.report_keywords, + 'st_keywords': self.st_keywords} + + @classmethod + def from_dict(cls, dct: Dict[str, bool]): + return cls(*tuple(dct.values())) + + @dataclass(init=False) + class Heuristics(ComplexSerializableType): + extracted_versions: List[str] + cpe_candidate_vendors: Optional[List[str]] = field(init=False) + cpe_matches: Optional[List[Tuple[float, CPE]]] + verified_cpe_matches: Optional[List[CPE]] + related_cves: Optional[List[str]] + labeled: bool + + def __init__(self, + extracted_versions: Optional[List[str]] = None, + cpe_matches: Optional[List[str]] = None, + verified_cpe_matches: Optional[List[str]] = None, + related_cves: Optional[List[CVE]] = None, + labeled: bool = False): + self.extracted_versions = extracted_versions + self.cpe_matches = cpe_matches + self.cpe_candidate_vendors = None + self.verified_cpe_matches = verified_cpe_matches + self.related_cves = related_cves + self.labeled = labeled + + def to_dict(self): + return {'extracted_versions': self.extracted_versions, 'cpe_matches': self.cpe_matches, 'verified_cpe_matches': self.verified_cpe_matches, 'related_cves': self.related_cves, 'labeled': self.labeled} + + @classmethod + def from_dict(cls, dct: Dict[str, str]): + return cls(*tuple(dct.values())) + + pandas_columns = ['dgst', 'name', 'status', 'category', 'manufacturer', 'scheme', 'security_level', + 'not_valid_before', 'not_valid_after', 'report_link', 'st_link', + 'manufacturer_web', 'extracted_versions', 'cpe_matches', 'verified_cpe_matches', + 'related_cves'] + + def __init__(self, status: str, category: str, name: str, manufacturer: str, scheme: str, + security_level: Union[str, set], not_valid_before: date, + not_valid_after: date, report_link: str, st_link: str, src: str, cert_link: Optional[str], + manufacturer_web: Optional[str], + protection_profiles: set, + maintainance_updates: set, + state: Optional[InternalState], + pdf_data: Optional[PdfData], + heuristics: Optional[Heuristics]): + super().__init__() + + self.status = status + self.category = category + self.name = helpers.sanitize_string(name) + self.manufacturer = helpers.sanitize_string(manufacturer) + self.scheme = scheme + self.security_level = helpers.sanitize_security_levels(security_level) + self.not_valid_before = helpers.sanitize_date(not_valid_before) + self.not_valid_after = helpers.sanitize_date(not_valid_after) + self.report_link = helpers.sanitize_link(report_link) + self.st_link = helpers.sanitize_link(st_link) + self.src = src + self.cert_link = helpers.sanitize_link(cert_link) + self.manufacturer_web = helpers.sanitize_link(manufacturer_web) + self.protection_profiles = protection_profiles + self.maintainance_updates = maintainance_updates + + if state is None: + state = self.InternalState() + self.state = state + + if pdf_data is None: + pdf_data = self.PdfData() + self.pdf_data = pdf_data + + if heuristics is None: + heuristics = self.Heuristics() + self.heuristics = heuristics + + @property + def dgst(self) -> str: + """ + Computes the primary key of the certificate using first 16 bytes of SHA-256 digest + """ + return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link) + + def __str__(self): + return self.manufacturer + ' ' + self.name + ' dgst: ' + self.dgst + + def to_pandas_tuple(self): + return self.dgst, self.name, self.status, self.category, self.manufacturer, self.scheme, self.security_level,\ + self.not_valid_before, self.not_valid_after, self.report_link, self.st_link, self.manufacturer_web, \ + self.heuristics.extracted_versions, self.heuristics.cpe_matches, self.heuristics.verified_cpe_matches, \ + self.heuristics.related_cves + + + def merge(self, other: 'CommonCriteriaCert'): + """ + Merges with other CC certificate. Assuming they come from different sources, e.g., csv and html. + Assuming that html source has better protection profiles, they overwrite CSV info + On other values (apart from maintainances, see TODO below) the sanity checks are made. + """ + if self != other: + logger.warning( + f'Attempting to merge divergent certificates: self[dgst]={self.dgst}, other[dgst]={other.dgst}') + + for att, val in vars(self).items(): + if not val: + setattr(self, att, getattr(other, att)) + elif self.src == 'csv' and other.src == 'html' and att == 'protection_profiles': + setattr(self, att, getattr(other, att)) + elif self.src == 'csv' and other.src == 'html' and att == 'maintainance_updates': + # TODO Fix me: This is a simplification. At the moment html contains more reliable info + setattr(self, att, getattr(other, att)) + elif att == 'src': + pass # This is expected + elif att == 'state': + setattr(self, att, getattr(other, att)) + else: + if getattr(self, att) != getattr(other, att): + logger.warning( + f'When merging certificates with dgst {self.dgst}, the following mismatch occured: Attribute={att}, self[{att}]={getattr(self, att)}, other[{att}]={getattr(other, att)}') + if self.src != other.src: + self.src = self.src + ' + ' + other.src + + @classmethod + def from_dict(cls, dct: Dict) -> 'CommonCriteriaCert': + new_dct = dct.copy() + new_dct['maintainance_updates'] = set(dct['maintainance_updates']) + new_dct['protection_profiles'] = set(dct['protection_profiles']) + return super(cls, CommonCriteriaCert).from_dict(new_dct) + + @classmethod + def from_html_row(cls, row: Tag, status: str, category: str) -> 'CommonCriteriaCert': + """ + Creates a CC certificate from html row + """ + + def _get_name(cell: Tag) -> str: + return list(cell.stripped_strings)[0] + + def _get_manufacturer(cell: Tag) -> Optional[str]: + if lst := list(cell.stripped_strings): + return lst[0] + else: + return None + + def _get_scheme(cell: Tag) -> str: + return list(cell.stripped_strings)[0] + + def _get_security_level(cell: Tag) -> set: + return set(cell.stripped_strings) + + def _get_manufacturer_web(cell: Tag) -> Optional[str]: + for link in cell.find_all('a'): + if link is not None and link.get('title') == 'Vendor\'s web site' and link.get('href') != 'http://': + return link.get('href') + return None + + def _get_protection_profiles(cell: Tag) -> set: + protection_profiles = set() + for link in list(cell.find_all('a')): + if link.get('href') is not None and '/ppfiles/' in link.get('href'): + protection_profiles.add(CommonCriteriaCert.ProtectionProfile(str(link.contents[0]), + CommonCriteriaCert.cc_url + link.get( + 'href'))) + return protection_profiles + + def _get_date(cell: Tag) -> date: + text = cell.get_text() + extracted_date = datetime.strptime( + text, '%Y-%m-%d').date() if text else None + return extracted_date + + def _get_report_st_links(cell: Tag) -> (str, str): + links = cell.find_all('a') + # TODO: Exception checks + assert links[1].get('title').startswith('Certification Report') + assert links[2].get('title').startswith('Security Target') + + report_link = CommonCriteriaCert.cc_url + links[1].get('href') + security_target_link = CommonCriteriaCert.cc_url + \ + links[2].get('href') + + return report_link, security_target_link + + def _get_cert_link(cell: Tag) -> Optional[str]: + links = cell.find_all('a') + return CommonCriteriaCert.cc_url + links[0].get('href') if links else None + + def _get_maintainance_div(cell: Tag) -> Optional[Tag]: + divs = cell.find_all('div') + for d in divs: + if d.find('div') and d.stripped_strings and list(d.stripped_strings)[0] == 'Maintenance Report(s)': + return d + return None + + def _get_maintainance_updates(main_div: Tag) -> set: + possible_updates = list(main_div.find_all('li')) + maintainance_updates = set() + for u in possible_updates: + text = list(u.stripped_strings)[0] + main_date = datetime.strptime(text.split( + ' ')[0], '%Y-%m-%d').date() if text else None + main_title = text.split('– ')[1] + main_report_link = None + main_st_link = None + links = u.find_all('a') + for l in links: + if l.get('title').startswith('Maintenance Report:'): + main_report_link = CommonCriteriaCert.cc_url + \ + l.get('href') + elif l.get('title').startswith('Maintenance ST'): + main_st_link = CommonCriteriaCert.cc_url + \ + l.get('href') + else: + logger.error('Unknown link in Maintenance part!') + maintainance_updates.add( + CommonCriteriaCert.MaintainanceReport(main_date, main_title, main_report_link, main_st_link)) + return maintainance_updates + + cells = list(row.find_all('td')) + if len(cells) != 7: + logger.error('Unexpected number of cells in CC html row.') + raise + + name = _get_name(cells[0]) + manufacturer = _get_manufacturer(cells[1]) + manufacturer_web = _get_manufacturer_web(cells[1]) + scheme = _get_scheme(cells[6]) + security_level = _get_security_level(cells[5]) + protection_profiles = _get_protection_profiles(cells[0]) + not_valid_before = _get_date(cells[3]) + not_valid_after = _get_date(cells[4]) + report_link, st_link = _get_report_st_links(cells[0]) + cert_link = _get_cert_link(cells[2]) + + maintainance_div = _get_maintainance_div(cells[0]) + maintainances = _get_maintainance_updates( + maintainance_div) if maintainance_div else set() + + return cls(status, category, name, manufacturer, scheme, security_level, not_valid_before, not_valid_after, + report_link, + st_link, 'html', cert_link, manufacturer_web, protection_profiles, maintainances, None, None, None) + + def set_local_paths(self, + report_pdf_dir: Optional[Union[str, Path]], + st_pdf_dir: Optional[Union[str, Path]], + report_txt_dir: Optional[Union[str, Path]], + st_txt_dir: Optional[Union[str, Path]]): + if report_pdf_dir is not None: + self.state.report_pdf_path = Path(report_pdf_dir) / (self.dgst + '.pdf') + if st_pdf_dir is not None: + self.state.st_pdf_path = Path(st_pdf_dir) / (self.dgst + '.pdf') + if report_txt_dir is not None: + self.state.report_txt_path = Path(report_txt_dir) / (self.dgst + '.txt') + if st_txt_dir is not None: + self.state.st_txt_path = Path(st_txt_dir) / (self.dgst + '.txt') + + @property + def best_cpe_match(self): + clean = [x for x in self.cpe_matching if len(x[0]) > 5] + cpe_match_ranking = [x[1] for x in clean] + argmax = cpe_match_ranking.index(max(cpe_match_ranking)) + return clean[argmax] + + @staticmethod + def download_pdf_report(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.download_file(cert.report_link, cert.state.report_pdf_path) + if exit_code != requests.codes.ok: + error_msg = f'failed to download report from {cert.report_link}, code: {exit_code}' + logger.error(f'Cert dgst: {cert.dgst} ' + error_msg) + cert.state.report_link_ok = False + cert.state.errors.append(error_msg) + return cert + + @staticmethod + def download_pdf_target(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.download_file(cert.st_link, cert.state.st_pdf_path) + if exit_code != requests.codes.ok: + error_msg = f'failed to download ST from {cert.report_link}, code: {exit_code}' + logger.error(f'Cert dgst: {cert.dgst}' + error_msg) + cert.state.st_link_ok = False + cert.state.errors.append(error_msg) + return cert + + def path_is_corrupted(self, local_path): + return not local_path.exists() or local_path.stat().st_size < constants.MIN_CORRECT_CERT_SIZE + + @staticmethod + def convert_report_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.convert_pdf_file(cert.state.report_pdf_path, cert.state.report_txt_path, ['-raw']) + if exit_code != constants.RETURNCODE_OK: + error_msg = 'failed to convert report pdf->txt' + logger.error(f'Cert dgst: {cert.dgst}' + error_msg) + cert.state.report_convert_ok = False + cert.state.errors.append(error_msg) + return cert + + @staticmethod + def convert_target_pdf(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + exit_code = helpers.convert_pdf_file(cert.state.st_pdf_path, cert.state.st_txt_path, ['-raw']) + if exit_code != constants.RETURNCODE_OK: + error_msg = 'failed to convert security target pdf->txt' + logger.error(f'Cert dgst: {cert.dgst}' + error_msg) + cert.state.st_convert_ok = False + cert.state.errors.append(error_msg) + return cert + + @staticmethod + def extract_st_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + response, cert.pdf_data.st_metadata = helpers.extract_pdf_metadata(cert.state.st_pdf_path) + if response != constants.RETURNCODE_OK: + cert.state.st_extract_ok = False + cert.state.errors.append(response) + return cert + + @staticmethod + def extract_report_pdf_metadata(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + response, cert.pdf_data.report_metadata = helpers.extract_pdf_metadata(cert.state.report_pdf_path) + if response != constants.RETURNCODE_OK: + cert.state.report_extract_ok = False + cert.state.errors.append(response) + return cert + + @staticmethod + def extract_st_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + cert.pdf_data.st_frontpage = dict() + + response_anssi, cert.pdf_data.st_frontpage['anssi'] = helpers.search_only_headers_anssi(cert.state.st_txt_path) + response_bsi, cert.pdf_data.st_frontpage['bsi'] = helpers.search_only_headers_bsi(cert.state.st_txt_path) + + if response_anssi != constants.RETURNCODE_OK: + cert.state.st_extract_ok = False + cert.state.errors.append(response_anssi) + if response_bsi != constants.RETURNCODE_OK: + cert.state.st_extract_ok = False + cert.state.errors.append(response_bsi) + + return cert + + @staticmethod + def extract_report_pdf_frontpage(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + cert.pdf_data.report_frontpage = dict() + response_bsi, cert.pdf_data.report_frontpage['bsi'] = helpers.search_only_headers_bsi( + cert.state.report_txt_path) + response_anssi, cert.pdf_data.report_frontpage['anssi'] = helpers.search_only_headers_anssi( + cert.state.report_txt_path) + + if response_anssi != constants.RETURNCODE_OK: + cert.state.report_extract_ok = False + cert.state.errors.append(response_anssi) + if response_bsi != constants.RETURNCODE_OK: + cert.state.report_extract_ok = False + cert.state.errors.append(response_bsi) + + return cert + + @staticmethod + def extract_report_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + response, cert.pdf_data.report_keywords = helpers.extract_keywords(cert.state.report_txt_path) + if response != constants.RETURNCODE_OK: + cert.state.report_extract_ok = False + return cert + + @staticmethod + def extract_st_pdf_keywords(cert: 'CommonCriteriaCert') -> 'CommonCriteriaCert': + response, cert.pdf_data.st_keywords = helpers.extract_keywords(cert.state.st_txt_path) + if response != constants.RETURNCODE_OK: + cert.state.st_extract_ok = False + cert.state.errors.append(response) + return cert + + def compute_heuristics_version(self): + """ + Will extract possible versions from the name + """ + at_least_something = r'(\b(\d)+\b)' + just_numbers = r'(\d{1,5})(\.\d{1,5})' + + without_version = r'(' + just_numbers + r'+)' + long_version = r'(' + r'(\bversion)\s*' + just_numbers + r'+)' + short_version = r'(' + r'\bv\s*' + just_numbers + r'+)' + full_regex_string = r'|'.join([without_version, short_version, long_version]) + normalizer = r'(\d+\.*)+' + + matched_strings = set([max(x, key=len) for x in re.findall(full_regex_string, self.name, re.IGNORECASE)]) + if not matched_strings: + matched_strings = set([max(x, key=len) for x in re.findall(at_least_something, self.name, re.IGNORECASE)]) + + if matched_strings: + self.heuristics.extracted_versions = [re.search(normalizer, x).group() for x in matched_strings] + else: + self.heuristics.extracted_versions = ['-'] + + def compute_heuristics_cpe_vendors(self, cpe_dataset: CPEDataset): + """ + With the help of the CPE dataset, will find CPE vendors that could match the given certificate vendor + """ + self.heuristics.cpe_candidate_vendors = cpe_dataset.get_candidate_list_of_vendors(self.manufacturer) + + def compute_heuristics_cpe_match(self, cpe_dataset: CPEDataset): + self.compute_heuristics_cpe_vendors(cpe_dataset) + self.heuristics.cpe_matches = cpe_dataset.get_cpe_matches(self.name, + self.heuristics.cpe_candidate_vendors, + self.heuristics.extracted_versions, + n_max_matches=constants.CPE_MAX_MATCHES, + threshold=constants.CPE_MATCHING_THRESHOLD) + + def compute_heuristics_related_cves(self, cve_dataset: CVEDataset): + if self.heuristics.verified_cpe_matches: + related_cves = [cve_dataset.get_cves_for_cpe(x.uri) for x in self.heuristics.verified_cpe_matches] + related_cves = list(filter(lambda x: x is not None, related_cves)) + if related_cves: + self.heuristics.related_cves = list(itertools.chain.from_iterable(related_cves)) + else: + self.heuristics.related_cves = None diff --git a/sec_certs/constants.py b/sec_certs/constants.py index db82b3ec..dd840640 100644 --- a/sec_certs/constants.py +++ b/sec_certs/constants.py @@ -44,7 +44,7 @@ TAG_PP_EDITOR = 'pp_editor' TAG_PP_REVIEWER = 'pp_reviewer' TAG_KEYWORDS = 'keywords' FIPS_NOT_AVAILABLE_CERT_SIZE = 10000 -FIPS_ALG_URL = 'https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/validation-search?searchMode=validation&page=' +FIPS_ALG_URL = 'https://csrc.nist.gov/projects/cryptographic-algorithm-validation-program/validation-search?searchMode=implementation&page=' FILE_ERRORS_STRATEGY = 'surrogateescape' STOP_ON_UNEXPECTED_NUMS = False diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py index 6e7b415e..cd42f9d0 100644 --- a/sec_certs/helpers.py +++ b/sec_certs/helpers.py @@ -138,8 +138,15 @@ def find_tables_iterative(file_text: str) -> List[int]: current_page += 1 if line.startswith('Table ') or line.startswith('Exhibit'): pages.add(current_page) + pages.add(current_page + 1) + if current_page > 2: + pages.add(current_page - 1) if not pages: logger.warning('No pages found') + for page in pages: + if page > current_page - 1: + return list(pages - {page}) + return list(pages) @@ -502,12 +509,15 @@ def extract_keywords(filepath: Path) -> Tuple[int, Optional[Dict[str, Dict[str, return constants.RETURNCODE_OK, processed_result -def analyze_matched_algs(data: Dict): +def plot_dataframe_graph(data: Dict, label: str, file_name: str, density: bool = False, cumulative: bool = False, bins: int = 50, log: bool = True, show: bool = True): pd_data = pd.Series(data) - pd_data.hist(bins=50) - plt.show() + pd_data.hist(bins=bins, label=label, density=density, cumulative=cumulative) + plt.savefig(file_name) + if show: + plt.show() - sorted_data = pd_data.value_counts(ascending=True) + if log: + sorted_data = pd_data.value_counts(ascending=True) logging.info(sorted_data.where(sorted_data > 1).dropna()) diff --git a/sec_certs/settings.yaml b/sec_certs/settings.yaml index 19e2811b..27f95f23 100644 --- a/sec_certs/settings.yaml +++ b/sec_certs/settings.yaml @@ -9,7 +9,7 @@ smallest_certificate_id_to_connect: year_difference_between_validations: description: During validation we don't connect certificates with validation dates difference higher than _this_ - value: 5 + value: 7 use_text_with_newlines_during_parsing: description: During keyword search, search in text with newlines value: true @@ -24,4 +24,11 @@ cc_cpe_max_matches: value: 20 cc_latest_snapshot: description: Url from where to fetch the latest snapshot of fully processed CC dataset - value: https://www.ajanovsky.cz/cc_latest_snapshot.json
\ No newline at end of file + value: https://www.ajanovsky.cz/cc_latest_snapshot.json +ignore_first_page: + description: During keyword search, first page usually contains addresses - ignore it. + value: true +cert_threshold: + description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty. + value: 5 + diff --git a/tests/data/test_fips_oop/algorithms.json b/tests/data/test_fips_oop/algorithms.json new file mode 100644 index 00000000..7845d93d --- /dev/null +++ b/tests/data/test_fips_oop/algorithms.json @@ -0,0 +1,513 @@ +{ + "_type": "FIPSAlgorithmDataset", + "certs": { + "2351": [ + { + "_type": "Algorithm", + "cert_id": "2351", + "date": "9/21/2018", + "implementation": "Apple CoreCrypto Kernel Module v9.0 for ARM (iOS12, A11 Bionic, Assembler_VNG)", + "type": "DRBG", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2351", + "date": "11/27/2015", + "implementation": "Apple iOS CoreCrypto Kernel Module (Optimized SHA, A6)", + "type": "HMAC", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2351", + "date": "1/27/2017", + "implementation": "OpenSSL using assembler for AES and SHA", + "type": "RSA", + "vendor": "Canonical Ltd." + }, + { + "_type": "Algorithm", + "cert_id": "2351", + "date": "1/19/2017", + "implementation": "Junos FIPS Version Junos 15.1 X49 - Dataplane_CN7020", + "type": "TDES", + "vendor": "Juniper Networks, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2351", + "date": "3/8/2013", + "implementation": "Samsung OpenSSL Cryptographic Module", + "type": "AES", + "vendor": "Samsung Electronics Co., Ltd" + }, + { + "_type": "Algorithm", + "cert_id": "2351", + "date": "3/7/2014", + "implementation": "Symantec PGP Cryptographic Engine", + "type": "SHS", + "vendor": "Symantec Corporation" + } + ], + "2352": [ + { + "_type": "Algorithm", + "cert_id": "2352", + "date": "9/21/2018", + "implementation": "Apple CoreCrypto Kernel Module v9.0 for ARM (iOS12, A10X Fusion, Assembler_VNG)", + "type": "DRBG", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2352", + "date": "3/8/2013", + "implementation": "AES-256 Core", + "type": "AES", + "vendor": "Altera Canada" + }, + { + "_type": "Algorithm", + "cert_id": "2352", + "date": "11/27/2015", + "implementation": "Apple iOS CoreCrypto Kernel Module (Optimized SHA, A6X)", + "type": "HMAC", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2352", + "date": "1/27/2017", + "implementation": "OpenSSL using support from Power ISA 2.07 for AES and SHA", + "type": "RSA", + "vendor": "Canonical Ltd." + }, + { + "_type": "Algorithm", + "cert_id": "2352", + "date": "1/19/2017", + "implementation": "Junos FIPS Version Junos 15.1 X49 - Dataplane_CN7130", + "type": "TDES", + "vendor": "Juniper Networks, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2352", + "date": "3/21/2014", + "implementation": "Karnak SHA in Hardware", + "type": "SHS", + "vendor": "Seagate Technology, LLC." + } + ], + "2600": [ + { + "_type": "Algorithm", + "cert_id": "2600", + "date": "12/15/2017", + "implementation": "Apple iOS CoreCrypto v8 Kernel Module (Generic Software Implementation)", + "type": "TDES", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2600", + "date": "6/10/2016", + "implementation": "IOS Common Cryptographic Module (IC2M) Algorithm Module", + "type": "HMAC", + "vendor": "Cisco Systems, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2600", + "date": "8/16/2013", + "implementation": "Blade System Virtual Connect", + "type": "AES", + "vendor": "Hewlett-Packard Development Company, L.P." + }, + { + "_type": "Algorithm", + "cert_id": "2600", + "date": "12/5/2014", + "implementation": "Cryptographic Security Kernel", + "type": "SHS", + "vendor": "IBM Corporation" + }, + { + "_type": "Algorithm", + "cert_id": "2600", + "date": "9/1/2017", + "implementation": "IBM z/OS(R) Cryptographic Services System SSL - 31bit", + "type": "RSA", + "vendor": "IBM Corporation" + } + ], + "2601": [ + { + "_type": "Algorithm", + "cert_id": "2601", + "date": "12/5/2014", + "implementation": "SHA256 Library on Canon MFP Security Chip", + "type": "SHS", + "vendor": "Canon Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2601", + "date": "8/16/2013", + "implementation": "Dell AppAssure Crypto Library", + "type": "AES", + "vendor": "Dell, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2601", + "date": "6/10/2016", + "implementation": "EFJ Communication Cryptographic Library", + "type": "HMAC", + "vendor": "EFJohnson Technologies" + }, + { + "_type": "Algorithm", + "cert_id": "2601", + "date": "9/1/2017", + "implementation": "IBM z/OS(R) Cryptographic Services System SSL - 64bit", + "type": "RSA", + "vendor": "IBM Corporation" + }, + { + "_type": "Algorithm", + "cert_id": "2601", + "date": "12/22/2017", + "implementation": "Oracle Linux 7 GnuTLS C Implementation", + "type": "TDES", + "vendor": "Oracle Corporation" + } + ], + "2602": [ + { + "_type": "Algorithm", + "cert_id": "2602", + "date": "12/22/2017", + "implementation": "Apple tvOS CoreCrypto Kernel Module v8.0 (Generic Software Implementation)", + "type": "TDES", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2602", + "date": "6/10/2016", + "implementation": "FIPS-ALGORITHMS.1.5.0v", + "type": "HMAC", + "vendor": "Mercury Systems" + }, + { + "_type": "Algorithm", + "cert_id": "2602", + "date": "8/16/2013", + "implementation": "RSA BSAFE\u00ae Crypto-J Software Module", + "type": "AES", + "vendor": "RSA Security, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2602", + "date": "12/5/2014", + "implementation": "SHA Library", + "type": "SHS", + "vendor": "Sage Microelectronics Corp" + }, + { + "_type": "Algorithm", + "cert_id": "2602", + "date": "9/1/2017", + "implementation": "Bouncy Castle FIPS Java API", + "type": "RSA", + "vendor": "Legion of the Bouncy Castle Inc." + } + ], + "2700": [ + { + "_type": "Algorithm", + "cert_id": "2700", + "date": "3/13/2015", + "implementation": "Apple OSX CoreCrypto Module (Generic, Xeon)", + "type": "SHS", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2700", + "date": "10/21/2016", + "implementation": "Axway OpenSSL", + "type": "HMAC", + "vendor": "Axway Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2700", + "date": "11/30/2017", + "implementation": "Brocade Fabric OS FIPS Cryptographic Module", + "type": "RSA", + "vendor": "Brocade Communications Systems LLC" + }, + { + "_type": "Algorithm", + "cert_id": "2700", + "date": "3/30/2018", + "implementation": "Junos OS 17.4R1-S1 - Dataplane", + "type": "TDES", + "vendor": "Juniper Networks, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2700", + "date": "11/29/2013", + "implementation": "VMware NSS Cryptographic Module", + "type": "AES", + "vendor": "VMware, Inc." + } + ], + "2701": [ + { + "_type": "Algorithm", + "cert_id": "2701", + "date": "3/30/2018", + "implementation": "Security Builder GSE-J Crypto Core", + "type": "TDES", + "vendor": "BlackBerry Certicom" + }, + { + "_type": "Algorithm", + "cert_id": "2701", + "date": "11/30/2017", + "implementation": "ngfips_rsa", + "type": "RSA", + "vendor": "Cavium, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2701", + "date": "10/28/2016", + "implementation": "Cisco_SSL_Implementation-1", + "type": "HMAC", + "vendor": "Cisco Systems, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2701", + "date": "3/13/2015", + "implementation": "RSA BSAFE\u00ae Crypto-J JSAFE and JCE Software Module", + "type": "SHS", + "vendor": "RSA, The Security Division of EMC" + }, + { + "_type": "Algorithm", + "cert_id": "2701", + "date": "11/29/2013", + "implementation": "VMware Cryptographic Module", + "type": "AES", + "vendor": "VMware, Inc." + } + ], + "2702": [ + { + "_type": "Algorithm", + "cert_id": "2702", + "date": "3/30/2018", + "implementation": "Security Builder GSE-J Crypto Core", + "type": "TDES", + "vendor": "BlackBerry Certicom" + }, + { + "_type": "Algorithm", + "cert_id": "2702", + "date": "11/30/2017", + "implementation": "DELPHI RSA2048 Signature Verification Algorithm Implementation", + "type": "RSA", + "vendor": "DELPHI" + }, + { + "_type": "Algorithm", + "cert_id": "2702", + "date": "12/6/2013", + "implementation": "RSA BSAFE Crypto-J", + "type": "AES", + "vendor": "McAfee, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "2702", + "date": "10/28/2016", + "implementation": "OpenSSL Crypto Library", + "type": "HMAC", + "vendor": "MikroM GmbH" + }, + { + "_type": "Algorithm", + "cert_id": "2702", + "date": "3/13/2015", + "implementation": "OpenSSL FIPS Object Module", + "type": "SHS", + "vendor": "OpenSSL Validation Services, Inc." + } + ], + "3415": [ + { + "_type": "Algorithm", + "cert_id": "3415", + "date": "1/26/2018", + "implementation": "Apple Secure Key Store CoreCrypto Module (Generic Software Implementation)", + "type": "HMAC", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3415", + "date": "6/5/2015", + "implementation": "Motorola Solutions Subscriber \u00b5Mace AES256", + "type": "AES", + "vendor": "Motorola Solutions Inc" + }, + { + "_type": "Algorithm", + "cert_id": "3415", + "date": "11/18/2016", + "implementation": "Secure Parser Library", + "type": "SHS", + "vendor": "Security First Corp." + } + ], + "3426": [ + { + "_type": "Algorithm", + "cert_id": "3426", + "date": "6/11/2015", + "implementation": "Apple iOS CoreCrypto Module (KeyWrap A8 32 bit)", + "type": "AES", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3426", + "date": "12/2/2016", + "implementation": "Apple iOS CoreCrypto Module (Generic)", + "type": "SHS", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3426", + "date": "1/26/2018", + "implementation": "Apple Secure Key Store CoreCrypto Module (VNG)", + "type": "HMAC", + "vendor": "Apple Inc." + } + ], + "3427": [ + { + "_type": "Algorithm", + "cert_id": "3427", + "date": "12/2/2016", + "implementation": "Apple iOS CoreCrypto Module (Generic)", + "type": "SHS", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3427", + "date": "1/26/2018", + "implementation": "Forcepoint NGFW FIPS Java API", + "type": "HMAC", + "vendor": "Forcepoint" + }, + { + "_type": "Algorithm", + "cert_id": "3427", + "date": "6/11/2015", + "implementation": "HP ESKM OpenSSL", + "type": "AES", + "vendor": "Hewlett Packard Enterprise" + } + ], + "3447": [ + { + "_type": "Algorithm", + "cert_id": "3447", + "date": "12/2/2016", + "implementation": "Apple OSX CoreCrypto Module (Optimized SHA nosse)", + "type": "SHS", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3447", + "date": "7/2/2015", + "implementation": "FireEye Algorithms Implementation", + "type": "AES", + "vendor": "FireEye, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3447", + "date": "2/9/2018", + "implementation": "OpenSSL (no AVX2/AVX/AESNI/SSSE3, x86_64, 64-bit library)", + "type": "HMAC", + "vendor": "Red Hat, Inc." + } + ], + "3451": [ + { + "_type": "Algorithm", + "cert_id": "3451", + "date": "12/2/2016", + "implementation": "Apple OSX CoreCrypto Module (Optimized SHA nosse)", + "type": "SHS", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3451", + "date": "7/2/2015", + "implementation": "OpenSSL FIPS Object Module", + "type": "AES", + "vendor": "OpenSSL Software Foundation, Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3451", + "date": "2/9/2018", + "implementation": "OpenSSL (no AVX2/AVX/AESNI, x86_64, 64-bit library)", + "type": "HMAC", + "vendor": "Red Hat, Inc." + } + ], + "3464": [ + { + "_type": "Algorithm", + "cert_id": "3464", + "date": "12/9/2016", + "implementation": "Apple OSX CoreCrypto Module (Generic)", + "type": "SHS", + "vendor": "Apple Inc." + }, + { + "_type": "Algorithm", + "cert_id": "3464", + "date": "7/10/2015", + "implementation": "Security Builder Linux Kernel Crypto Core", + "type": "AES", + "vendor": "Certicom Corp." + }, + { + "_type": "Algorithm", + "cert_id": "3464", + "date": "2/9/2018", + "implementation": "HPE Secure Encryption Engine v1.1", + "type": "HMAC", + "vendor": "Hewlett-Packard Development Company, L.P." + } + ] + } +}
\ No newline at end of file diff --git a/tests/fips_test_utils.py b/tests/fips_test_utils.py new file mode 100644 index 00000000..94fb406a --- /dev/null +++ b/tests/fips_test_utils.py @@ -0,0 +1,51 @@ +from typing import List +from pathlib import Path + +def generate_html(ids: List[str], path: Path): + def generate_entry(certificate_id: str) -> str: + return f''' + <tr id="cert-row-0"> + <td class="text-center"> + <a href="/projects/cryptographic-module-validation-program/certificate/3898" id="cert-number-link-0">{certificate_id}</a> + </td> + </tr> + ''' + + html_head = ''' + <!DOCTYPE html> + <html lang="en-us" xml:lang="en-us"> + <head> + <meta charset="utf-8" /> + <title>Cryptographic Module Validation Program | CSRC</title> + <meta http-equiv="content-type" content="text/html; charset=UTF-8" /> + <meta http-equiv="content-style-type" content="text/css" /> + <meta http-equiv="content-script-type" content="text/javascript" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="msapplication-config" content="/CSRC/Media/images/favicons/browserconfig.xml" /> + <meta name="theme-color" content="#000000" /> + <meta name="google-site-verification" content="xbrnrVYDgLD-Bd64xHLCt4XsPXzUhQ-4lGMj4TdUUTA" /> + </head> + ''' + rows = "" + for cert_id in ids: + rows += f"\n{generate_entry(cert_id)}\n" + html_body = f''' + <body> + <table class="table table-striped table-condensed publications-table table-bordered" id="searchResultsTable"> + <thead> + <tr> + <th class="text-center">Certificate Number</th> + <th class="text-center">Vendor Name</th> + <th class="text-center">Module Name</th> + <th class="text-center">Module Type</th> + <th class="text-center">Validation Date</th> + </tr> + </thead> + <tbody> + {rows} + </tbody> + </table> + </body> + ''' + with open(path, 'w') as f: + f.write(f"{html_head}\n{html_body}\n") diff --git a/tests/settings_test.yaml b/tests/settings_test.yaml new file mode 100644 index 00000000..9b07a8be --- /dev/null +++ b/tests/settings_test.yaml @@ -0,0 +1,19 @@ +--- +smallest_certificate_id_to_connect: + description: During validation we don't connect certificates with number lower than + _this_ to connections + value: 40 +year_difference_between_validations: + description: During validation we don't connect certificates with validation dates + difference higher than _this_ + value: 7 +use_text_with_newlines_during_parsing: + description: During keyword search, search in text with newlines + value: true +ignore_first_page: + description: During keyword search, first page usually contains addresses - ignore it. + value: true +cert_threshold: + description: Used with --higher-precision-results. Determines the amount of mismatched algorithms to be considered faulty. + value: 5 + diff --git a/tests/test_fips_oop.py b/tests/test_fips_oop.py new file mode 100644 index 00000000..107f0c5d --- /dev/null +++ b/tests/test_fips_oop.py @@ -0,0 +1,134 @@ +from unittest import TestCase +from pathlib import Path +from tempfile import TemporaryDirectory + +from sec_certs.dataset import FIPSDataset, FIPSAlgorithmDataset +from sec_certs.configuration import config +from fips_test_utils import generate_html + + + +def _set_up_dataset(td, certs): + dataset = FIPSDataset({}, Path(td), 'test_dataset', 'fips_test_dataset') + generate_html(certs, td + '/test_search.html') + dataset.get_certs_from_web(test=td + '/test_search.html', update_json=False) + return dataset + + +def _set_up_dataset_for_full(td, certs): + dataset = _set_up_dataset(td, certs) + dataset.convert_all_pdfs() + dataset.extract_keywords() + dataset.extract_certs_from_tables(high_precision=True) + dataset.algorithms = FIPSAlgorithmDataset.from_json(Path(__file__).parent / 'data/test_fips_oop/algorithms.json') + dataset.finalize_results() + return dataset + + +class TestFipsOOP(TestCase): + def setUp(self) -> None: + self.data_dir: Path = Path(__file__).parent / 'data' / 'test_fips_oop' + self.dataset = FIPSDataset({}, self.data_dir, 'test_dataset', 'fips_test_dataset') + self.certs_to_parse = [ + ['3099', '2549', '2484', '3038', '2472', '2435', '2471', '1930'], # openSUSE chunk + ['23', '24', '25', '26'], + ['3095', '3651', '3093', '3090', '3197', '3196', '3089', '3195', '3480', '3615', '3194', '3091', '3690', + '3644', '3527', '3094', '3544', '3096', '3092'], # microsoft chunk + ['2630', '2721', '2997', '2441', '2711', '2633', '2798', '3613', '3733', '2908', '2446', '2742', '2447'], + # redhat chunk + ['3850', '2779', '2860', '2665', '1883', '3518', '3141', '2590'], # Document signing chunk + ['3493', '3495', '3711', '3176', '3488', '3126', '3269', '3524', '3220', '2398', '3543', '2676', '3313', + '3363', '3608', '3158'], # Chunk referencing openSSL FIPS Object Module SE + ] + config.load(Path(__file__).parent / 'settings_test.yaml') + + def test_size(self): + for certs in self.certs_to_parse: + with TemporaryDirectory() as td: + dataset = _set_up_dataset(td, certs) + self.assertEqual(len(dataset.certs), len(certs), "Wrong number of parsed certs") + + def test_connections_microsoft(self): + certs = self.certs_to_parse[2] + with TemporaryDirectory() as td: + dataset = _set_up_dataset_for_full(td, certs) + + self.assertEqual(set(dataset.certs['3095'].processed.connections), {x for x in ['3093', '3096', '3094']}) + self.assertEqual(set(dataset.certs['3651'].processed.connections), {x for x in ['3615']}) + self.assertEqual(set(dataset.certs['3093'].processed.connections), {x for x in ['3090', '3091']}) + self.assertEqual(set(dataset.certs['3090'].processed.connections), {x for x in ['3089']}) + self.assertEqual(set(dataset.certs['3197'].processed.connections), + {x for x in ['3195', '3096', '3196', '3644', '3651']}) + self.assertEqual(set(dataset.certs['3196'].processed.connections), + {x for x in ['3194', '3091', '3480', '3615']}) + self.assertEqual(set(dataset.certs['3089'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['3195'].processed.connections), {x for x in ['3194', '3091', '3480']}) + self.assertEqual(set(dataset.certs['3480'].processed.connections), {x for x in ['3089']}) + self.assertEqual(set(dataset.certs['3615'].processed.connections), {x for x in ['3089']}) + self.assertEqual(set(dataset.certs['3194'].processed.connections), {x for x in ['3089']}) + self.assertEqual(set(dataset.certs['3091'].processed.connections), {x for x in ['3089']}) + self.assertEqual(set(dataset.certs['3690'].processed.connections), {x for x in ['3644', '3196', '3651']}) + self.assertEqual(set(dataset.certs['3644'].processed.connections), {x for x in ['3615']}) + self.assertEqual(set(dataset.certs['3527'].processed.connections), {x for x in ['3090', '3091']}) + self.assertEqual(set(dataset.certs['3094'].processed.connections), {x for x in ['3090', '3091']}) + self.assertEqual(set(dataset.certs['3544'].processed.connections), {x for x in ['3093', '3096', '3527']}) + self.assertEqual(set(dataset.certs['3096'].processed.connections), + {x for x in ['3090', '3194', '3091', '3480']}) + self.assertEqual(set(dataset.certs['3092'].processed.connections), + {x for x in ['3093', '3195', '3096', '3644', '3651']}) + + def test_connections_redhat(self): + certs = self.certs_to_parse[3] + with TemporaryDirectory() as td: + dataset = _set_up_dataset_for_full(td, certs) + self.assertEqual(set(dataset.certs['2630'].processed.connections), {x for x in ['2441']}) + self.assertEqual(set(dataset.certs['2633'].processed.connections), {x for x in ['2441']}) + self.assertEqual(set(dataset.certs['2441'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['2997'].processed.connections), {x for x in ['2711']}) + self.assertEqual(set(dataset.certs['2446'].processed.connections), {x for x in ['2441']}) + self.assertEqual(set(dataset.certs['2447'].processed.connections), {x for x in ['2441']}) + self.assertEqual(set(dataset.certs['3733'].processed.connections), {x for x in ['2441']}) + self.assertEqual(set(dataset.certs['2441'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['2711'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['2908'].processed.connections), {x for x in ['2711']}) + self.assertEqual(set(dataset.certs['3613'].processed.connections), {x for x in ['2997']}) + self.assertEqual(set(dataset.certs['2721'].processed.connections), {x for x in ['2441', '2711']}) + self.assertEqual(set(dataset.certs['2798'].processed.connections), {x for x in ['2721', '2711']}) + self.assertEqual(set(dataset.certs['2711'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['2997'].processed.connections), {x for x in ['2711']}) + self.assertEqual(set(dataset.certs['2742'].processed.connections), {x for x in ['2721', '2711']}) + self.assertEqual(set(dataset.certs['2721'].processed.connections), {x for x in ['2441', '2711']}) + + def test_docusign_chunk(self): + certs = self.certs_to_parse[4] + with TemporaryDirectory() as td: + dataset = _set_up_dataset_for_full(td, certs) + self.assertEqual(set(dataset.certs['3850'].processed.connections), {x for x in ['3518', '1883']}) + self.assertEqual(set(dataset.certs['2779'].processed.connections), {x for x in ['1883']}) + self.assertEqual(set(dataset.certs['2860'].processed.connections), {x for x in ['1883']}) + self.assertEqual(set(dataset.certs['2665'].processed.connections), {x for x in ['1883']}) + self.assertEqual(set(dataset.certs['1883'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['3518'].processed.connections), {x for x in ['1883']}) + self.assertEqual(set(dataset.certs['3141'].processed.connections), {x for x in ['1883']}) + self.assertEqual(set(dataset.certs['2590'].processed.connections), {x for x in ['1883']}) + + def test_openssl_chunk(self): + certs = self.certs_to_parse[5] + with TemporaryDirectory() as td: + dataset = _set_up_dataset_for_full(td, certs) + self.assertEqual(set(dataset.certs['3493'].processed.connections), {x for x in ['2398']}) + self.assertEqual(set(dataset.certs['3495'].processed.connections), {x for x in ['2398']}) + self.assertEqual(set(dataset.certs['3711'].processed.connections), {x for x in ['3220']}) + self.assertEqual(set(dataset.certs['3176'].processed.connections), {x for x in ['2398']}) + self.assertEqual(set(dataset.certs['3488'].processed.connections), {x for x in ['2398']}) + self.assertEqual(set(dataset.certs['3126'].processed.connections), {x for x in ['3126', '2398']}) + self.assertEqual(set(dataset.certs['3269'].processed.connections), {x for x in ['3269', '3220']}) + self.assertEqual(set(dataset.certs['3524'].processed.connections), {x for x in ['3220']}) + self.assertEqual(set(dataset.certs['3220'].processed.connections), {x for x in ['3220', '2398']}) + self.assertEqual(set(dataset.certs['2398'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['3543'].processed.connections), {x for x in ['2398']}) + self.assertEqual(set(dataset.certs['2676'].processed.connections), {x for x in ['2398']}) + self.assertEqual(set(dataset.certs['3313'].processed.connections), {x for x in ['3313', '3220']}) + self.assertEqual(set(dataset.certs['3363'].processed.connections), {x for x in []}) + self.assertEqual(set(dataset.certs['3608'].processed.connections), {x for x in ['2398']}) + self.assertEqual(set(dataset.certs['3158'].processed.connections), {x for x in ['2398']}) |