aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorAdam Janovsky2021-10-19 16:42:23 +0200
committerAdam Janovsky2021-10-19 16:42:23 +0200
commit37263b6f7f9acfe90e1fc084717886e62e054253 (patch)
treeebcb34d260f3ccd81254a724b20e9dff6249607f
parent03f145fa63a290fbdb731a692aa5eff6c84526a3 (diff)
downloadsec-certs-37263b6f7f9acfe90e1fc084717886e62e054253.tar.gz
sec-certs-37263b6f7f9acfe90e1fc084717886e62e054253.tar.zst
sec-certs-37263b6f7f9acfe90e1fc084717886e62e054253.zip
rename certificate -> sample and split cve,cpe dataset from sample
-rw-r--r--sec_certs/cert_rules.py74
-rw-r--r--sec_certs/dataset/common_criteria.py18
-rw-r--r--sec_certs/dataset/cpe.py51
-rw-r--r--sec_certs/dataset/cve.py120
-rw-r--r--sec_certs/dataset/dataset.py4
-rw-r--r--sec_certs/dataset/fips.py8
-rw-r--r--sec_certs/dataset/fips_algorithm.py2
-rw-r--r--sec_certs/dataset/protection_profile.py2
-rw-r--r--sec_certs/helpers.py6
-rw-r--r--sec_certs/model/cpe_matching.py8
-rw-r--r--sec_certs/model/cve_matching.py2
-rw-r--r--sec_certs/sample/__init__.py (renamed from sec_certs/certificate/__init__.py)0
-rw-r--r--sec_certs/sample/cc_maintenance_update.py (renamed from sec_certs/certificate/cc_maintenance_update.py)2
-rw-r--r--sec_certs/sample/certificate.py (renamed from sec_certs/certificate/certificate.py)0
-rw-r--r--sec_certs/sample/common_criteria.py (renamed from sec_certs/certificate/common_criteria.py)21
-rw-r--r--sec_certs/sample/cpe.py42
-rw-r--r--sec_certs/sample/cve.py122
-rw-r--r--sec_certs/sample/fips.py (renamed from sec_certs/certificate/fips.py)5
-rw-r--r--sec_certs/sample/protection_profile.py (renamed from sec_certs/certificate/protection_profile.py)0
-rw-r--r--tests/data/test_cc_oop/cc_products_active.html4
-rw-r--r--tests/test_cc_heuristics.py12
-rw-r--r--tests/test_cc_oop.py12
-rw-r--r--tests/test_cc_txt_processing.py2
23 files changed, 268 insertions, 249 deletions
diff --git a/sec_certs/cert_rules.py b/sec_certs/cert_rules.py
index ba857300..b7c87835 100644
--- a/sec_certs/cert_rules.py
+++ b/sec_certs/cert_rules.py
@@ -322,28 +322,28 @@ rules_fips_remove_algorithm_ids = [
# --- HMAC(-SHA)(-1) - (bits) (method) ((hardware/firmware cert) #id) ---
# + added (and #id) everywhere
- r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*and[\s#]*\d+)?",
- r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*and[\s#]*\d+)?",
- r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*and[\s#]*\d+)?",
- r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*and[\s#]*\d+)?",
+ r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*and[\s#]*\d+)?",
+ r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*and[\s#]*\d+)?",
+ r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*and[\s#]*\d+)?",
+ r"HMAC(?:[- –]*SHA)?(?:[- –]*1)?[– -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?\(?(?: |hardware|firmware)*?[\s(\[]*?(?:#|cert\.?|Cert\.?|Certificate|sample)?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*and[\s#]*\d+)?",
# --- same as above, without hw or fw ---
- r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
- r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
- r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
- r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
+ r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
+ r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
+ r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
+ r"HMAC(?:-SHA)?(?:-1)?[ -]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[, ]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
# --- SHS/A - (bits) (method) ((cert #) numbers) ---
- r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
- r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
- r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
- r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
+ r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
+ r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
+ r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
+ r"SH[SA][-– 123]*(?:;|\/|160|224|256|384|512)?(?:[\s(\[]*?(?:KAT|[Bb]yte [Oo]riented)*?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?\[#?\d+\])?(?:[\s#]*?and[\s#]*?\d+)?",
# --- RSA (bits) (method) ((cert #)) ---
- r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
- r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
- r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
- r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
+ r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
+ r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
+ r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
+ r"RSA(?:[-– ]*(?:;|\/|512|768|1024|1280|1536|2048|3072|4096|8192)\s\(\[]*?(?:(?:;|\/|KAT|Verify|PSS|\s)*?)?[\s,]*?[\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
# --- RSA (SSA) (PKCS) (version) (#) ---
r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{4})?",
@@ -352,34 +352,34 @@ rules_fips_remove_algorithm_ids = [
r"(?:RSA)?[-– ]?(?:SSA)?[- ]?PKCS\s?#?\d(?:-[Vv]1_5| [Vv]1[-_]5)?[\s#]*?(\d{1})?",
# --- AES (bits) (method) ((cert #)) ---
- r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
- r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
- r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
- r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"AES[-– ]*((?: |;|\/|bit|key|128|192|256|CBC)*(?: |\/|;|[Dd]ecrypt|[Ee]ncrypt|KAT|CMAC|CTR|GCM|IV|CBC)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})(?:\)?[\s#]*?\[#?\d+\])?(?:[\s#]*?and[\s#]*?(\d+))?",
# --- Diffie Helman (CVL) ((cert #)) ---
- r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{4})",
- r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{3})",
- r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{2})",
- r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{1})",
+ r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{4})",
+ r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{3})",
+ r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{2})",
+ r"Diffie[-– ]*Hellman[,\s(\[]*?(?:CVL|\s)*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?[\s#]*?(\d{1})",
# --- DRBG (bits) (method) (cert #) ---
- r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
- r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
- r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
- r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
+ r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
+ r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
+ r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
+ r"DRBG[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
# --- DES (bits) (method) (cert #)
- r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*?and[\s#]*?(\d+))?",
- r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*?and[\s#]*?(\d+))?",
- r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*?and[\s#]*?(\d+))?",
- r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{4})(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{3})(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{2})(?:[\s#]*?and[\s#]*?(\d+))?",
+ r"DES[ –-]*((?:;|\/|160|224|256|384|512)?(?:;|\/| |[Dd]ecrypt|[Ee]ncrypt|KAT|CBC|(?:\d(?: and \d)? keying options?))*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)*?[\s#]*?)?[\s#]*?(\d{1})(?:[\s#]*?and[\s#]*?(\d+))?",
# --- DSA (bits) (method) (cert #)
- r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
- r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
- r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
- r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|certificate|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
+ r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{4})",
+ r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{3})",
+ r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{2})",
+ r"DSA[ –-]*((?:;|\/|160|224|256|384|512)?(?: |[Dd]ecrypt|[Ee]ncrypt|KAT)*?[,\s(\[]*?(?:#|cert\.?|sample|Cert\.?|Certificate)?[\s#]*?)?[\s#]*?(\d{1})",
# --- platforms (#)+ - this is used in modification history ---
r"[Pp]latforms? #\d+(?:#\d+|,| |-|and)*[^\n]*",
@@ -397,7 +397,7 @@ rules_fips_remove_algorithm_ids = [
r"PAA[: #]*?\d{1}",
# --- (#) Type ---
- r"(?:#|cert\.?|certificate|Cert\.?|Certificate)[\s#]*?(\d+)?\s*?(?:AES|SHS|SHA|RSA|HMAC|Diffie-Hellman|DRBG|DES|CVL)",
+ r"(?:#|cert\.?|sample|Cert\.?|Certificate)[\s#]*?(\d+)?\s*?(?:AES|SHS|SHA|RSA|HMAC|Diffie-Hellman|DRBG|DES|CVL)",
# --- PKCS (#) ---
r"PKCS[\s]?#?\d+",
diff --git a/sec_certs/dataset/common_criteria.py b/sec_certs/dataset/common_criteria.py
index a1fd4cde..8ac5c414 100644
--- a/sec_certs/dataset/common_criteria.py
+++ b/sec_certs/dataset/common_criteria.py
@@ -16,15 +16,17 @@ from bs4 import Tag, BeautifulSoup
from tqdm import tqdm
from sec_certs import helpers as helpers, parallel_processing as cert_processing, constants as constants
-from sec_certs.dataset.cpe import CPEDataset, CPE
-from sec_certs.dataset.cve import CVEDataset, CVE
+from sec_certs.dataset.cpe import CPEDataset
+from sec_certs.sample.cpe import CPE
+from sec_certs.dataset.cve import CVEDataset
+from sec_certs.sample.cve import CVE
from sec_certs.dataset.dataset import Dataset, logger
from sec_certs.serialization import ComplexSerializableType, serialize, CustomJSONDecoder
-from sec_certs.certificate.common_criteria import CommonCriteriaCert
+from sec_certs.sample.common_criteria import CommonCriteriaCert
from sec_certs.dataset.protection_profile import ProtectionProfileDataset
-from sec_certs.certificate.protection_profile import ProtectionProfile
+from sec_certs.sample.protection_profile import ProtectionProfile
from sec_certs.configuration import config
-from sec_certs.certificate.cc_maintenance_update import CommonCriteriaMaintenanceUpdate
+from sec_certs.sample.cc_maintenance_update import CommonCriteriaMaintenanceUpdate
class CCDataset(Dataset, ComplexSerializableType):
@@ -450,7 +452,7 @@ class CCDataset(Dataset, ComplexSerializableType):
logger.error('Attempting to download pdfs while not having csv/html meta-sources parsed. Returning.')
return
- logger.info('Downloading CC certificate reports')
+ logger.info('Downloading CC sample reports')
self._download_reports(fresh)
logger.info('Downloading CC security targets')
@@ -487,7 +489,7 @@ class CCDataset(Dataset, ComplexSerializableType):
logger.info('Attempting to convert pdf while not having them downloaded. Returning.')
return
- logger.info('Converting CC certificate reports to .txt')
+ logger.info('Converting CC sample reports to .txt')
self._convert_reports_to_txt(fresh)
logger.info('Converting CC security targets to .txt')
@@ -613,7 +615,7 @@ class CCDataset(Dataset, ComplexSerializableType):
cert.compute_heuristics_cert_lab()
def _compute_cert_ids(self):
- logger.info('Deriving information about certificate ids from pdf scan.')
+ logger.info('Deriving information about sample ids from pdf scan.')
certs_to_process = [x for x in self if x.state.report_is_ok_to_analyze()]
for cert in certs_to_process:
cert.compute_heuristics_cert_id()
diff --git a/sec_certs/dataset/cpe.py b/sec_certs/dataset/cpe.py
index 1892e593..ad065866 100644
--- a/sec_certs/dataset/cpe.py
+++ b/sec_certs/dataset/cpe.py
@@ -4,14 +4,14 @@ import json
from typing import Optional, List, Dict, Tuple, Set, Union, ClassVar
import itertools
import re
-from rapidfuzz import process, fuzz
+from rapidfuzz import fuzz
import tempfile
from pathlib import Path
import zipfile
import operator
import sec_certs.helpers as helpers
-from sec_certs.serialization import ComplexSerializableType
+from sec_certs.sample.cpe import CPE
import pandas as pd
import xml.etree.ElementTree as ET
@@ -19,44 +19,6 @@ import xml.etree.ElementTree as ET
logger = logging.getLogger(__name__)
-@dataclass(init=False)
-class CPE(ComplexSerializableType):
- uri: str
- title: str
- version: str
- vendor: str
- item_name: str
- pandas_columns: ClassVar[List[str]] = ['uri', 'vendor', 'item_name', 'version', 'title']
-
- def __init__(self, uri: Optional[str] = None, title: Optional[str] = None):
- self.uri = uri
- self.title = title
-
- if self.uri:
- self.vendor = ' '.join(self.uri.split(':')[3].split('_'))
- self.item_name = ' '.join(self.uri.split(':')[4].split('_'))
- self.version = self.uri.split(':')[5]
-
- def __lt__(self, other: 'CPE'):
- return self.title < other.title
-
- @property
- def serialized_attributes(self) -> List[str]:
- return ['uri', 'title']
-
- @property
- def pandas_tuple(self):
- return self.uri, self.vendor, self.item_name, self.version, self.title
-
- def __hash__(self):
- return hash(self.uri)
-
- def __eq__(self, other):
- if not isinstance(other, CPE):
- return False
- return self.uri == other.uri
-
-
def build_cpe_uri_to_title_dict(input_xml_filepath: str, output_filepath: str):
"""
Will parse CPE XML file into dictionary cpe_uri: cpe_title and dump the dict into json
@@ -249,10 +211,10 @@ class CPEDataset:
def get_candidate_vendor_version_pairs(self, cert_candidate_cpe_vendors: List[str], cert_candidate_versions: List[str]) -> Optional[List[Tuple[str, str]]]:
"""
- Given parameters, will return Pairs (cpe_vendor, cpe_version) that should are relevant to a given certificate
+ Given parameters, will return Pairs (cpe_vendor, cpe_version) that should are relevant to a given sample
Parameters
- :param cert_candidate_cpe_vendors: list of CPE vendors relevant to a certificate
- :param cert_candidate_versions: List of versions heuristically extracted from the certificate name
+ :param cert_candidate_cpe_vendors: list of CPE vendors relevant to a sample
+ :param cert_candidate_versions: List of versions heuristically extracted from the sample name
:return: List of tuples (cpe_vendor, cpe_version) that can be used in the lookup table to search the CPE dataset.
"""
@@ -273,7 +235,8 @@ class CPEDataset:
candidate_vendor_version_pairs.extend([(vendor, x) for x in matched_cpe_versions])
return candidate_vendor_version_pairs
- def get_candidate_cpe_items(self, cert_candidate_cpe_vendors: List[str], cert_candidate_versions: List[str]) -> Optional[List[CPE]]:
+ def get_candidate_cpe_items(self, cert_candidate_cpe_vendors: List[str], cert_candidate_versions: List[str]) -> Optional[List[
+ CPE]]:
candidate_vendor_version_pairs = self.get_candidate_vendor_version_pairs(cert_candidate_cpe_vendors, cert_candidate_versions)
if not candidate_vendor_version_pairs:
diff --git a/sec_certs/dataset/cve.py b/sec_certs/dataset/cve.py
index 6de58f0c..0fb687f7 100644
--- a/sec_certs/dataset/cve.py
+++ b/sec_certs/dataset/cve.py
@@ -7,135 +7,19 @@ import zipfile
import logging
import glob
import json
-from dateutil.parser import isoparse
-import copy
import pandas as pd
-import numpy as np
from sec_certs.parallel_processing import process_parallel
import sec_certs.constants as constants
import sec_certs.helpers as helpers
+from sec_certs.sample.cve import CVE
from sec_certs.serialization import ComplexSerializableType, CustomJSONDecoder, CustomJSONEncoder
from sec_certs.configuration import config
logger = logging.getLogger(__name__)
-
-@dataclass(init=False)
-class CVE(ComplexSerializableType):
- @dataclass(eq=True)
- class Impact(ComplexSerializableType):
- base_score: float
- severity: str
- explotability_score: float
- impact_score: float
-
- @classmethod
- def from_nist_dict(cls, dct: Dict):
- """
- Will load Impact from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1
- """
- if not dct['impact']:
- return cls(0, '', 0, 0)
- elif 'baseMetricV3' in dct['impact']:
- return cls(dct['impact']['baseMetricV3']['cvssV3']['baseScore'],
- dct['impact']['baseMetricV3']['cvssV3']['baseSeverity'],
- dct['impact']['baseMetricV3']['exploitabilityScore'],
- dct['impact']['baseMetricV3']['impactScore'])
- elif 'baseMetricV2' in dct['impact']:
- return cls(dct['impact']['baseMetricV2']['cvssV2']['baseScore'],
- dct['impact']['baseMetricV2']['severity'],
- dct['impact']['baseMetricV2']['exploitabilityScore'],
- dct['impact']['baseMetricV2']['impactScore'])
-
- cve_id: str
- vulnerable_cpes: List[str]
- vulnerable_certs: List[str]
- impact: Impact
- published_date: Optional[datetime.datetime]
- tokenized: Optional[str]
-
- pandas_columns: Final[List[str]] = ('cve_id', 'vulnerable_cpes', 'vulnerable_certs', 'base_score', 'severity',
- 'explotability_score', 'impact_score', 'published_date', 'description')
-
- def __init__(self, cve_id: str, vulnerable_cpes: List[str], vulnerable_certs: Optional[List[str]], impact: Impact,
- published_date: str, tokenized=None):
- super().__init__()
- self.cve_id = cve_id
- self.vulnerable_cpes = vulnerable_cpes
-
- self.vulnerable_certs = vulnerable_certs
- if not self.vulnerable_certs:
- self.vulnerable_certs = []
-
- self.impact = impact
- self.published_date = isoparse(published_date)
- self.tokenized = tokenized
-
- def __hash__(self):
- return hash(self.cve_id)
-
- def __eq__(self, other):
- if not isinstance(other, CVE):
- return False
- return self.cve_id == other.cve_id
-
- def __lt__(self, other):
- if not isinstance(other, CVE):
- raise ValueError(f'Cannot compare CVE with {type(other)} type.')
- self_year = int(self.cve_id.split('-')[1])
- self_id = int(self.cve_id.split('-')[2])
- other_year = int(other.cve_id.split('-')[1])
- other_id = int(other.cve_id.split('-')[2])
-
- return self_year < other_year if self_year != other_year else self_id < other_id
-
- @property
- def serialized_attributes(self) -> List[str]:
- all_vars = copy.deepcopy(super().serialized_attributes)
- all_vars.remove('tokenized')
- return all_vars
-
- @classmethod
- def from_nist_dict(cls, dct: Dict) -> 'CVE':
- """
- Will load CVE from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1
- """
- def get_vulnerable_cpes_from_nist_dict(dct: Dict) -> List[str]:
- def get_vulnerable_cpes_from_node(node: Dict) -> List[str]:
- cpe_uris = []
- if 'children' in node:
- for child in node['children']:
- cpe_uris += get_vulnerable_cpes_from_node(child)
- if 'cpe_match' in node:
- lst = node['cpe_match']
- for x in lst:
- if x['vulnerable']:
- cpe_uris.append(x['cpe23Uri'])
- return cpe_uris
-
- vulnerable_cpes = []
- for node in dct['configurations']['nodes']:
- vulnerable_cpes.extend(get_vulnerable_cpes_from_node(node))
-
- return vulnerable_cpes
-
- cve_id = dct['cve']['CVE_data_meta']['ID']
- impact = cls.Impact.from_nist_dict(dct)
- vulnerable_cpes = get_vulnerable_cpes_from_nist_dict(dct)
- vulnerable_certs = None
- published_date = dct['publishedDate']
-
- description = dct['cve']['description']['description_data'][0]['value']
-
- return cls(cve_id, vulnerable_cpes, vulnerable_certs, impact, published_date, description)
-
- def to_pandas_tuple(self):
- return (self.cve_id, self.vulnerable_cpes, self.vulnerable_certs, self.impact.base_score, self.impact.severity,
- self.impact.explotability_score, self.impact.impact_score, self.published_date)
-
@dataclass(eq=True)
class CVEDataset(ComplexSerializableType):
cves: Dict[str, CVE]
@@ -228,7 +112,7 @@ class CVEDataset(ComplexSerializableType):
def filter_related_cpes(self, relevant_cpe_uris: Set[str]):
"""
Since each of the CVEs is related to many CPEs, the dataset size explodes (serialized). For certificates,
- only CPEs within certificate dataset are relevant. This function modifies all CVE elements. Specifically, it
+ only CPEs within sample dataset are relevant. This function modifies all CVE elements. Specifically, it
deletes all CPE records unless they are part of relevant_cpe_uris.
:param relevant_cpe_uris: List of relevant CPE uris to keep in CVE dataset.
"""
diff --git a/sec_certs/dataset/dataset.py b/sec_certs/dataset/dataset.py
index 57c9a793..49cb2c24 100644
--- a/sec_certs/dataset/dataset.py
+++ b/sec_certs/dataset/dataset.py
@@ -12,7 +12,7 @@ import sec_certs.helpers as helpers
import sec_certs.constants as constants
import sec_certs.parallel_processing as cert_processing
-from sec_certs.certificate.certificate import Certificate
+from sec_certs.sample.certificate import Certificate
from sec_certs.serialization import CustomJSONDecoder, CustomJSONEncoder
from sec_certs.configuration import config
from sec_certs.serialization import serialize
@@ -154,7 +154,7 @@ class Dataset(ABC):
return cpe_dataset
def _compute_candidate_versions(self):
- logger.info('Computing heuristics: possible product versions in certificate name')
+ logger.info('Computing heuristics: possible product versions in sample name')
for cert in self:
cert.compute_heuristics_version()
diff --git a/sec_certs/dataset/fips.py b/sec_certs/dataset/fips.py
index 354aec8d..c769f27e 100644
--- a/sec_certs/dataset/fips.py
+++ b/sec_certs/dataset/fips.py
@@ -14,7 +14,7 @@ from sec_certs.configuration import config
from sec_certs.dataset.dataset import Dataset, logger
from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset
from sec_certs.serialization import ComplexSerializableType, serialize
-from sec_certs.certificate.fips import FIPSCertificate
+from sec_certs.sample.fips import FIPSCertificate
from sec_certs.dataset.cpe import CPEDataset
@@ -141,7 +141,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
@serialize
def convert_all_pdfs(self):
- logger.info('Converting FIPS certificate reports to .txt')
+ logger.info('Converting FIPS sample reports to .txt')
tuples = [
(cert, self.policies_dir / f"{cert.cert_id}.pdf", self.policies_dir / f"{cert.cert_id}.pdf.txt")
for cert in self.certs.values()
@@ -176,7 +176,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
self.download_all_pdfs()
def _get_certificates_from_html(self, html_file: Path, update: bool = False) -> None:
- logger.info(f"Getting certificate ids from {html_file}")
+ logger.info(f"Getting sample ids from {html_file}")
with open(html_file, "r", encoding="utf-8") as handle:
html = BeautifulSoup(handle.read(), "html.parser")
@@ -252,7 +252,7 @@ class FIPSDataset(Dataset, ComplexSerializableType):
# Download files containing all available module certs (always)
self.prepare_dataset(test, update)
- logger.info("Downloading certificate html and security policies")
+ logger.info("Downloading sample html and security policies")
self.download_neccessary_files()
if not no_download_algorithms:
diff --git a/sec_certs/dataset/fips_algorithm.py b/sec_certs/dataset/fips_algorithm.py
index 42f1b22e..b957033b 100644
--- a/sec_certs/dataset/fips_algorithm.py
+++ b/sec_certs/dataset/fips_algorithm.py
@@ -9,7 +9,7 @@ import sec_certs.helpers
from sec_certs import helpers as helpers, constants as constants, parallel_processing as cert_processing
from sec_certs.dataset.dataset import Dataset
from sec_certs.serialization import ComplexSerializableType, CustomJSONEncoder, CustomJSONDecoder
-from sec_certs.certificate.fips import FIPSCertificate
+from sec_certs.sample.fips import FIPSCertificate
from sec_certs.configuration import config
diff --git a/sec_certs/dataset/protection_profile.py b/sec_certs/dataset/protection_profile.py
index 51617b1c..558faa23 100644
--- a/sec_certs/dataset/protection_profile.py
+++ b/sec_certs/dataset/protection_profile.py
@@ -6,7 +6,7 @@ import logging
import tempfile
import sec_certs.helpers as helpers
-from sec_certs.certificate.protection_profile import ProtectionProfile
+from sec_certs.sample.protection_profile import ProtectionProfile
logger = logging.getLogger(__name__)
diff --git a/sec_certs/helpers.py b/sec_certs/helpers.py
index b8928c9a..14f4cf7e 100644
--- a/sec_certs/helpers.py
+++ b/sec_certs/helpers.py
@@ -92,7 +92,7 @@ def sanitize_string(record: str) -> Union[str, None]:
if not record:
return None
else:
- # TODO: There is a certificate with name 'ATMEL Secure Microcontroller AT90SC12872RCFT &#x2f; AT90SC12836RCFT rev. I &amp;&#x23;38&#x3b; J' that has to be unescaped twice
+ # TODO: There is a sample with name 'ATMEL Secure Microcontroller AT90SC12872RCFT &#x2f; AT90SC12836RCFT rev. I &amp;&#x23;38&#x3b; J' that has to be unescaped twice
string = html.unescape(html.unescape(record)).replace('\n', '')
return ' '.join(string.split())
@@ -457,7 +457,7 @@ def search_only_headers_bsi(filepath: Path):
items_found[constants.TAG_DEVELOPER] = normalize_match_string(developer)
items_found[constants.TAG_CERT_LAB] = 'BSI'
- # Process page with more detailed certificate info
+ # Process page with more detailed sample info
# PP Conformance, Functionality, Assurance
rules_certificate_third = ['PP Conformance: (.+)Functionality: (.+)Assurance: (.+)The IT Product identified']
@@ -705,7 +705,7 @@ def gen_dict_extract(dct: Dict, searched_key: Hashable = 'count'):
def compute_heuristics_version(cert_name: str) -> List[str]:
"""
- Will extract possible versions from the name of certificate
+ Will extract possible versions from the name of sample
"""
at_least_something = r'(\b(\d)+\b)'
just_numbers = r'(\d{1,5})(\.\d{1,5})'
diff --git a/sec_certs/model/cpe_matching.py b/sec_certs/model/cpe_matching.py
index ec5d8ce1..93d50df3 100644
--- a/sec_certs/model/cpe_matching.py
+++ b/sec_certs/model/cpe_matching.py
@@ -1,6 +1,6 @@
from sklearn.base import BaseEstimator
from typing import Dict, Tuple, Set, List, Optional, Union
-from sec_certs.dataset.cpe import CPE
+from sec_certs.sample.cpe import CPE
import sec_certs.helpers as helpers
import tqdm
import itertools
@@ -191,10 +191,10 @@ class CPEClassifier(BaseEstimator):
def get_candidate_vendor_version_pairs(self, cert_candidate_cpe_vendors: List[str], cert_candidate_versions: List[str]) -> Optional[List[Tuple[str, str]]]:
"""
- Given parameters, will return Pairs (cpe_vendor, cpe_version) that should are relevant to a given certificate
+ Given parameters, will return Pairs (cpe_vendor, cpe_version) that should are relevant to a given sample
Parameters
- :param cert_candidate_cpe_vendors: list of CPE vendors relevant to a certificate
- :param cert_candidate_versions: List of versions heuristically extracted from the certificate name
+ :param cert_candidate_cpe_vendors: list of CPE vendors relevant to a sample
+ :param cert_candidate_versions: List of versions heuristically extracted from the sample name
:return: List of tuples (cpe_vendor, cpe_version) that can be used in the lookup table to search the CPE dataset.
"""
diff --git a/sec_certs/model/cve_matching.py b/sec_certs/model/cve_matching.py
index f89a5510..274b2aa3 100644
--- a/sec_certs/model/cve_matching.py
+++ b/sec_certs/model/cve_matching.py
@@ -8,7 +8,7 @@ from sklearn.base import BaseEstimator
from sklearn.neighbors import NearestNeighbors
import seaborn as sns
import sec_certs.helpers as helpers
-from sec_certs.dataset.cve import CVE
+from sec_certs.sample.cve import CVE
class VulnClassifier(BaseEstimator):
diff --git a/sec_certs/certificate/__init__.py b/sec_certs/sample/__init__.py
index e69de29b..e69de29b 100644
--- a/sec_certs/certificate/__init__.py
+++ b/sec_certs/sample/__init__.py
diff --git a/sec_certs/certificate/cc_maintenance_update.py b/sec_certs/sample/cc_maintenance_update.py
index 7e4db349..ba5ef541 100644
--- a/sec_certs/certificate/cc_maintenance_update.py
+++ b/sec_certs/sample/cc_maintenance_update.py
@@ -3,7 +3,7 @@ from typing import Optional, Dict, List, ClassVar, Tuple
from datetime import date
import sec_certs.helpers as helpers
-from sec_certs.certificate.common_criteria import CommonCriteriaCert
+from sec_certs.sample.common_criteria import CommonCriteriaCert
from sec_certs.serialization import ComplexSerializableType
logger = logging.getLogger(__name__)
diff --git a/sec_certs/certificate/certificate.py b/sec_certs/sample/certificate.py
index ddcd2cb7..ddcd2cb7 100644
--- a/sec_certs/certificate/certificate.py
+++ b/sec_certs/sample/certificate.py
diff --git a/sec_certs/certificate/common_criteria.py b/sec_certs/sample/common_criteria.py
index 4c99a93d..556ab43f 100644
--- a/sec_certs/certificate/common_criteria.py
+++ b/sec_certs/sample/common_criteria.py
@@ -13,13 +13,16 @@ from bs4 import Tag
import numpy as np
from sec_certs import helpers, constants as constants
-from sec_certs.certificate.certificate import Certificate, logger
-from sec_certs.dataset.cpe import CPE, CPEDataset
-from sec_certs.dataset.cve import CVE, CVEDataset
+from sec_certs.sample.certificate import Certificate, logger
+from sec_certs.dataset.cpe import CPEDataset
+from sec_certs.sample.cpe import CPE
+from sec_certs.dataset.cve import CVEDataset
+from sec_certs.sample.cve import CVE
from sec_certs.serialization import ComplexSerializableType
-from sec_certs.certificate.protection_profile import ProtectionProfile
+from sec_certs.sample.protection_profile import ProtectionProfile
from sec_certs.configuration import config
+
class CommonCriteriaCert(Certificate, ComplexSerializableType):
cc_url = 'http://www.commoncriteriaportal.org'
empty_st_url = 'http://www.commoncriteriaportal.org/files/epfiles/'
@@ -253,7 +256,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
@property
def dgst(self) -> str:
"""
- Computes the primary key of the certificate using first 16 bytes of SHA-256 digest
+ Computes the primary key of the sample using first 16 bytes of SHA-256 digest
"""
return helpers.get_first_16_bytes_sha256(self.category + self.name + self.report_link)
@@ -272,7 +275,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
def merge(self, other: 'CommonCriteriaCert', other_source: Optional[str] = None):
"""
- Merges with other CC certificate. Assuming they come from different sources, e.g., csv and html.
+ Merges with other CC sample. Assuming they come from different sources, e.g., csv and html.
Assuming that html source has better protection profiles, they overwrite CSV info
On other values (apart from maintainances, see TODO below) the sanity checks are made.
"""
@@ -304,7 +307,7 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
@classmethod
def from_html_row(cls, row: Tag, status: str, category: str) -> 'CommonCriteriaCert':
"""
- Creates a CC certificate from html row
+ Creates a CC sample from html row
"""
def _get_name(cell: Tag) -> str:
@@ -571,13 +574,13 @@ class CommonCriteriaCert(Certificate, ComplexSerializableType):
def compute_heuristics_cert_lab(self):
if not self.pdf_data:
- logger.error('Cannot compute certificate lab when pdf files were not processed.')
+ logger.error('Cannot compute sample lab when pdf files were not processed.')
return
self.heuristics.cert_lab = self.pdf_data.cert_lab
def compute_heuristics_cert_id(self):
if not self.pdf_data:
- logger.error('Cannot compute certificate id when pdf files were not processed.')
+ logger.error('Cannot compute sample id when pdf files were not processed.')
return
self.heuristics.cert_id = self.pdf_data.cert_id
diff --git a/sec_certs/sample/cpe.py b/sec_certs/sample/cpe.py
new file mode 100644
index 00000000..bc88b74c
--- /dev/null
+++ b/sec_certs/sample/cpe.py
@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+from typing import ClassVar, List, Optional
+
+from sec_certs.serialization import ComplexSerializableType
+
+
+@dataclass(init=False)
+class CPE(ComplexSerializableType):
+ uri: str
+ title: str
+ version: str
+ vendor: str
+ item_name: str
+ pandas_columns: ClassVar[List[str]] = ['uri', 'vendor', 'item_name', 'version', 'title']
+
+ def __init__(self, uri: Optional[str] = None, title: Optional[str] = None):
+ self.uri = uri
+ self.title = title
+
+ if self.uri:
+ self.vendor = ' '.join(self.uri.split(':')[3].split('_'))
+ self.item_name = ' '.join(self.uri.split(':')[4].split('_'))
+ self.version = self.uri.split(':')[5]
+
+ def __lt__(self, other: 'CPE'):
+ return self.title < other.title
+
+ @property
+ def serialized_attributes(self) -> List[str]:
+ return ['uri', 'title']
+
+ @property
+ def pandas_tuple(self):
+ return self.uri, self.vendor, self.item_name, self.version, self.title
+
+ def __hash__(self):
+ return hash(self.uri)
+
+ def __eq__(self, other):
+ if not isinstance(other, CPE):
+ return False
+ return self.uri == other.uri \ No newline at end of file
diff --git a/sec_certs/sample/cve.py b/sec_certs/sample/cve.py
new file mode 100644
index 00000000..47d9f347
--- /dev/null
+++ b/sec_certs/sample/cve.py
@@ -0,0 +1,122 @@
+import copy
+import datetime
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Final
+
+from dateutil.parser import isoparse
+
+from sec_certs.serialization import ComplexSerializableType
+
+
+@dataclass(init=False)
+class CVE(ComplexSerializableType):
+ @dataclass(eq=True)
+ class Impact(ComplexSerializableType):
+ base_score: float
+ severity: str
+ explotability_score: float
+ impact_score: float
+
+ @classmethod
+ def from_nist_dict(cls, dct: Dict):
+ """
+ Will load Impact from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1
+ """
+ if not dct['impact']:
+ return cls(0, '', 0, 0)
+ elif 'baseMetricV3' in dct['impact']:
+ return cls(dct['impact']['baseMetricV3']['cvssV3']['baseScore'],
+ dct['impact']['baseMetricV3']['cvssV3']['baseSeverity'],
+ dct['impact']['baseMetricV3']['exploitabilityScore'],
+ dct['impact']['baseMetricV3']['impactScore'])
+ elif 'baseMetricV2' in dct['impact']:
+ return cls(dct['impact']['baseMetricV2']['cvssV2']['baseScore'],
+ dct['impact']['baseMetricV2']['severity'],
+ dct['impact']['baseMetricV2']['exploitabilityScore'],
+ dct['impact']['baseMetricV2']['impactScore'])
+
+ cve_id: str
+ vulnerable_cpes: List[str]
+ vulnerable_certs: List[str]
+ impact: Impact
+ published_date: Optional[datetime.datetime]
+ tokenized: Optional[str]
+
+ pandas_columns: Final[List[str]] = ('cve_id', 'vulnerable_cpes', 'vulnerable_certs', 'base_score', 'severity',
+ 'explotability_score', 'impact_score', 'published_date', 'description')
+
+ def __init__(self, cve_id: str, vulnerable_cpes: List[str], vulnerable_certs: Optional[List[str]], impact: Impact,
+ published_date: str, tokenized=None):
+ super().__init__()
+ self.cve_id = cve_id
+ self.vulnerable_cpes = vulnerable_cpes
+
+ self.vulnerable_certs = vulnerable_certs
+ if not self.vulnerable_certs:
+ self.vulnerable_certs = []
+
+ self.impact = impact
+ self.published_date = isoparse(published_date)
+ self.tokenized = tokenized
+
+ def __hash__(self):
+ return hash(self.cve_id)
+
+ def __eq__(self, other):
+ if not isinstance(other, CVE):
+ return False
+ return self.cve_id == other.cve_id
+
+ def __lt__(self, other):
+ if not isinstance(other, CVE):
+ raise ValueError(f'Cannot compare CVE with {type(other)} type.')
+ self_year = int(self.cve_id.split('-')[1])
+ self_id = int(self.cve_id.split('-')[2])
+ other_year = int(other.cve_id.split('-')[1])
+ other_id = int(other.cve_id.split('-')[2])
+
+ return self_year < other_year if self_year != other_year else self_id < other_id
+
+ @property
+ def serialized_attributes(self) -> List[str]:
+ all_vars = copy.deepcopy(super().serialized_attributes)
+ all_vars.remove('tokenized')
+ return all_vars
+
+ @classmethod
+ def from_nist_dict(cls, dct: Dict) -> 'CVE':
+ """
+ Will load CVE from dictionary defined at https://nvd.nist.gov/feeds/json/cve/1.1
+ """
+ def get_vulnerable_cpes_from_nist_dict(dct: Dict) -> List[str]:
+ def get_vulnerable_cpes_from_node(node: Dict) -> List[str]:
+ cpe_uris = []
+ if 'children' in node:
+ for child in node['children']:
+ cpe_uris += get_vulnerable_cpes_from_node(child)
+ if 'cpe_match' in node:
+ lst = node['cpe_match']
+ for x in lst:
+ if x['vulnerable']:
+ cpe_uris.append(x['cpe23Uri'])
+ return cpe_uris
+
+ vulnerable_cpes = []
+ for node in dct['configurations']['nodes']:
+ vulnerable_cpes.extend(get_vulnerable_cpes_from_node(node))
+
+ return vulnerable_cpes
+
+ cve_id = dct['cve']['CVE_data_meta']['ID']
+ impact = cls.Impact.from_nist_dict(dct)
+ vulnerable_cpes = get_vulnerable_cpes_from_nist_dict(dct)
+ vulnerable_certs = None
+ published_date = dct['publishedDate']
+
+ description = dct['cve']['description']['description_data'][0]['value']
+
+ return cls(cve_id, vulnerable_cpes, vulnerable_certs, impact, published_date, description)
+
+ def to_pandas_tuple(self):
+ return (self.cve_id, self.vulnerable_cpes, self.vulnerable_certs, self.impact.base_score, self.impact.severity,
+ self.impact.explotability_score, self.impact.impact_score, self.published_date) \ No newline at end of file
diff --git a/sec_certs/certificate/fips.py b/sec_certs/sample/fips.py
index 97f17c07..4373fc5b 100644
--- a/sec_certs/certificate/fips.py
+++ b/sec_certs/sample/fips.py
@@ -13,12 +13,13 @@ from tabula import read_pdf
import sec_certs.constants
from sec_certs import helpers, dataset, constants as constants
from sec_certs.cert_rules import fips_common_rules, REGEXEC_SEP, fips_rules
-from sec_certs.certificate.certificate import Certificate, logger
+from sec_certs.sample.certificate import Certificate, logger
from sec_certs.configuration import config
from sec_certs.constants import LINE_SEPARATOR
from sec_certs.helpers import save_modified_cert_file, normalize_match_string, load_cert_file
from sec_certs.serialization import ComplexSerializableType
-from sec_certs.dataset.cpe import CPE, CPEDataset
+from sec_certs.dataset.cpe import CPEDataset
+from sec_certs.sample.cpe import CPE
class FIPSCertificate(Certificate, ComplexSerializableType):
diff --git a/sec_certs/certificate/protection_profile.py b/sec_certs/sample/protection_profile.py
index 1bca2c31..1bca2c31 100644
--- a/sec_certs/certificate/protection_profile.py
+++ b/sec_certs/sample/protection_profile.py
diff --git a/tests/data/test_cc_oop/cc_products_active.html b/tests/data/test_cc_oop/cc_products_active.html
index 6c44e3c0..ca76dc18 100644
--- a/tests/data/test_cc_oop/cc_products_active.html
+++ b/tests/data/test_cc_oop/cc_products_active.html
@@ -376,12 +376,12 @@
<p>
The CCDB has approved a resolution to limit the validity of mutually recognized CC certificates over time.&nbsp;
Certificates will remain on the CPL for five years.&nbsp;
- Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date of certificate issuance) will be moved to an Archive list on the CCRA portal
+ Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date of sample issuance) will be moved to an Archive list on the CCRA portal
</p>
<p>
The CCDB has approved a resolution to limit the validity of mutually recognized CC certificates over time.&nbsp;
Certificates will remain on the CPL for five years.&nbsp;
- Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date of certificate issuance) will be moved to an Archive list on the CCRA portal.&nbsp;
+ Effective 1 June 2019, certificates with an expired validity period (that is, 5 years or more from the date of sample issuance) will be moved to an Archive list on the CCRA portal.&nbsp;
Certificates fully recognized under the CCRA must claim conformance to either a collaborative Protection Profile (cPP) or EAL1-2.&nbsp;
<p>
-->
diff --git a/tests/test_cc_heuristics.py b/tests/test_cc_heuristics.py
index 94a054ff..236abfd2 100644
--- a/tests/test_cc_heuristics.py
+++ b/tests/test_cc_heuristics.py
@@ -3,10 +3,12 @@ import datetime
import tempfile
from unittest import TestCase
from sec_certs.dataset.common_criteria import CCDataset
-from sec_certs.certificate.common_criteria import CommonCriteriaCert
-from sec_certs.certificate.protection_profile import ProtectionProfile
-from sec_certs.dataset.cpe import CPEDataset, CPE
-from sec_certs.dataset.cve import CVEDataset, CVE
+from sec_certs.sample.common_criteria import CommonCriteriaCert
+from sec_certs.sample.protection_profile import ProtectionProfile
+from sec_certs.dataset.cpe import CPEDataset
+from sec_certs.sample.cpe import CPE
+from sec_certs.dataset.cve import CVEDataset
+from sec_certs.sample.cve import CVE
from pathlib import Path
from typing import ClassVar, Dict
import shutil
@@ -98,7 +100,7 @@ class TestCommonCriteriaHeuristics(TestCase):
self.assertCountEqual(evaluated_cves, self.cc_dset['ebd276cca70fd723'].heuristics.related_cves, 'The computed CVEs do not match the excpected CVEs')
def test_version_extraction(self):
- self.assertEqual(self.cc_dset['ebd276cca70fd723'].heuristics.extracted_versions, ['8.2'], 'The version extracted from the certificate does not match the template')
+ self.assertEqual(self.cc_dset['ebd276cca70fd723'].heuristics.extracted_versions, ['8.2'], 'The version extracted from the sample does not match the template')
new_cert = CommonCriteriaCert('', '', 'IDOneClassIC Card : ID-One Cosmo 64 RSA v5.4 and applet IDOneClassIC v1.0 embedded on P5CT072VOP', '', '',
'', None, None, '', '', '', '', set(), set(), None, None, None)
new_cert.compute_heuristics_version()
diff --git a/tests/test_cc_oop.py b/tests/test_cc_oop.py
index 54fcbd4b..e8595e46 100644
--- a/tests/test_cc_oop.py
+++ b/tests/test_cc_oop.py
@@ -9,8 +9,8 @@ import shutil
import os
from sec_certs.dataset.common_criteria import CCDataset
-from sec_certs.certificate.common_criteria import CommonCriteriaCert
-from sec_certs.certificate.protection_profile import ProtectionProfile
+from sec_certs.sample.common_criteria import CommonCriteriaCert
+from sec_certs.sample.protection_profile import ProtectionProfile
import sec_certs.helpers as helpers
import sec_certs.constants as constants
@@ -103,7 +103,7 @@ class TestCommonCriteriaOOP(TestCase):
actual_report_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.report_pdf_path) for key, val in dset.certs.items()}
actual_target_pdf_hashes = {key: helpers.get_sha256_filepath(val.state.st_pdf_path) for key, val in dset.certs.items()}
- self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (certificate report) do not the template')
+ self.assertEqual(actual_report_pdf_hashes, self.template_report_pdf_hashes, 'Hashes of downloaded pdfs (sample report) do not the template')
self.assertEqual(actual_target_pdf_hashes, self.template_target_pdf_hashes, 'Hashes of downloaded pdfs (security target) do not match the template')
self.assertTrue(dset['309ac2fd7f2dcf17'].state.report_txt_path.exists())
@@ -122,7 +122,7 @@ class TestCommonCriteriaOOP(TestCase):
self.fictional_cert.to_json(tmp.name)
self.assertTrue(filecmp.cmp(self.test_data_dir / 'fictional_cert.json',
tmp.name),
- 'The certificate serialized to json differs from a template.')
+ 'The sample serialized to json differs from a template.')
def test_dataset_to_json(self):
with NamedTemporaryFile('w') as tmp:
@@ -134,7 +134,7 @@ class TestCommonCriteriaOOP(TestCase):
def test_cert_from_json(self):
self.assertEqual(self.fictional_cert,
CommonCriteriaCert.from_json(self.test_data_dir / 'fictional_cert.json'),
- 'The certificate serialized from json differs from a template.')
+ 'The sample serialized from json differs from a template.')
def test_dataset_from_json(self):
self.assertEqual(self.template_dataset,
@@ -166,7 +166,7 @@ class TestCommonCriteriaOOP(TestCase):
self.assertEqual(len(dset), 2, 'The dataset should contain 2 files.')
- self.assertTrue(self.crt_one in dset, 'The dataset does not contain the template certificate.')
+ self.assertTrue(self.crt_one in dset, 'The dataset does not contain the template sample.')
self.assertEqual(dset, self.template_dataset, 'The loaded dataset does not match the template dataset.')
def test_download_csv_html_files(self):
diff --git a/tests/test_cc_txt_processing.py b/tests/test_cc_txt_processing.py
index 3a677e8a..07afcff7 100644
--- a/tests/test_cc_txt_processing.py
+++ b/tests/test_cc_txt_processing.py
@@ -6,7 +6,7 @@ import tempfile
import shutil
from sec_certs.dataset.common_criteria import CCDataset
-from sec_certs.certificate.common_criteria import CommonCriteriaCert
+from sec_certs.sample.common_criteria import CommonCriteriaCert
class TestCommonCriteriaHeuristics(TestCase):