From a53f0f71ab0c741d274693aa03e34c74df10d5a2 Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Fri, 3 Mar 2023 09:58:29 +0100 Subject: move data to package, be more explicit about package_data --- .../manual_annotations/test.csv | 1172 ++++++++++++++++++++ .../manual_annotations/train.csv | 101 ++ .../manual_annotations/valid.csv | 101 ++ .../outdated_manually_annotated_references.csv | 242 ++++ src/sec_certs/data/reference_annotations/readme.md | 51 + .../data/reference_annotations/split/test.json | 780 +++++++++++++ .../data/reference_annotations/split/train.json | 466 ++++++++ .../data/reference_annotations/split/valid.json | 312 ++++++ 8 files changed, 3225 insertions(+) create mode 100644 src/sec_certs/data/reference_annotations/manual_annotations/test.csv create mode 100644 src/sec_certs/data/reference_annotations/manual_annotations/train.csv create mode 100644 src/sec_certs/data/reference_annotations/manual_annotations/valid.csv create mode 100644 src/sec_certs/data/reference_annotations/outdated_manually_annotated_references.csv create mode 100644 src/sec_certs/data/reference_annotations/readme.md create mode 100644 src/sec_certs/data/reference_annotations/split/test.json create mode 100644 src/sec_certs/data/reference_annotations/split/train.json create mode 100644 src/sec_certs/data/reference_annotations/split/valid.json (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/test.csv b/src/sec_certs/data/reference_annotations/manual_annotations/test.csv new file mode 100644 index 00000000..fd43357f --- /dev/null +++ b/src/sec_certs/data/reference_annotations/manual_annotations/test.csv @@ -0,0 +1,1172 @@ +dgst,referenced_cert_id,source,label,comment +00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None +00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None +00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None +00b539d629e60965,CCEVS-VR-02-0025,None,None,None +00eb3e5c7a4537b1,BSI-DSZ-CC-0266-2005,None,None,None +00eb3e5c7a4537b1,BSI-DSZ-CC-0322-2005,None,None,None +00eb3e5c7a4537b1,BSI-DSZ-CC-0399-2007,None,None,None +017f3cea8275a3e1,ANSSI-CC-2015/59,None,None,None +017f3cea8275a3e1,ANSSI-CC-2017/23,None,None,None +01ce6f422bec823d,BSI-DSZ-CC-1040-2019,None,None,None +01dc77d7f8f94d97,ANSSI-CC-2016/78,None,None,None +01dc77d7f8f94d97,ANSSI-CC-2017/39,None,None,None +01e5c6d31f763f56,BSI-DSZ-CC-0917-2014,None,None,None +0208e6b60a0b1a32,ANSSI-CC-2016/01,None,None,None +02c6fbcd90dcf05a,BSI-DSZ-CC-1107-V3-2022,None,None,None +02cb3c385865ad2c,BSI-DSZ-CC-1184-2022,None,None,None +0319b03323f257d1,BSI-DSZ-CC-1118-2020,None,None,None +0319b03323f257d1,BSI-DSZ-CC-1121-2019,None,None,None +032faef0207b715f,BSI-DSZ-CC-0837-V2-2014,None,None,None +0349c893b81f7c1a,BSI-DSZ-CC-1079-V2-2020,None,None,None +04c5a5a66fb7f89a,BSI-DSZ-CC-0827-V2-2014,None,None,None +04cb360502da6492,BSI-DSZ-CC-0344-2005,None,None,None +054bd8196e015197,ANSSI-CC-2011/07,None,None,None +05d96aada7bb839c,BSI-DSZ-CC-0223-2003,None,None,None +06b24bc51eb69188,BSI-DSZ-CC-0813-2012,None,None,None +06e70fa70393d60f,ANSSI-CC-2017/07,None,None,None +06e70fa70393d60f,BSI-DSZ-CC-0891-V2-2016,None,None,None +075ddd00e07fa7ed,ANSSI-CC-2020/76,None,None,None +075ddd00e07fa7ed,BSI-DSZ-CC-1136-2021,None,None,None +07ce7e4844f62c5c,ANSSI-CC-2017/73,None,None,None +07ce7e4844f62c5c,BSI-DSZ-CC-1070-2018,None,None,None +080aa6966d757f05,ANSSI-CC-2010/02,None,None,None +082380caaa153ee6,BSI-DSZ-CC-0891-V2-2016,None,None,None +08408cbc394e7421,ANSSI-CC-2017/59,None,None,None +08408cbc394e7421,BSI-DSZ-CC-0782-V4-2018,None,None,None +09220515a7edf3f9,BSI-DSZ-CC-1110-V2-2019,None,None,None +096b05114ee5e09f,BSI-DSZ-CC-0410-2007,None,None,None +096b05114ee5e09f,BSI-DSZ-CC-0411-2007,None,None,None +096ccb6bcb3d2731,BSI-DSZ-CC-0564-2009,None,None,None +09b07fbbb24e1b04,BSI-DSZ-CC-0964-2015,None,None,None +09f539aedefa9756,ANSSI-CC-2009/41,None,None,None +0a6e50445a906052,BSI-DSZ-CC-0891-V2-2016,None,None,None +0a6e50445a906052,BSI-DSZ-CC-0945-2017,None,None,None +0aad0176e35b3749,BSI-DSZ-CC-0477-2007,None,None,None +0b0648a9b9a26747,BSI-DSZ-CC-0223-2003,None,None,None +0bd651fcbb86f2a9,ANSSI-CC-2020/74,None,None,None +0bd651fcbb86f2a9,BSI-DSZ-CC-1136-2021,None,None,None +0c9e215997d96f44,BSI-DSZ-CC-0555-2009,None,None,None +0cd406e7c5013b74,BSI-DSZ-CC-0304-2006,None,None,None +0d4a95a8b52b787b,BSI-DSZ-CC-0829-2012,None,None,None +0da9d81c9f563df5,BSI-DSZ-CC-0382-2007,None,None,None +0ee132dec087b772,ANSSI-CC-2011/07,None,None,None +0ef56cba2362e463,ANSSI-CC-2015/42,None,None,None +0f1b31ac93459d1f,BSI-DSZ-CC-0720-2011,None,None,None +0f684159ad31f883,BSI-DSZ-CC-0555-2009,None,None,None +0fe0bb1054f49b7f,CCEVS-VR-04-0092,None,None,None +0ff82b1cc24b3105,BSI-DSZ-CC-0302-2005,None,None,None +10233188161519ee,ANSSI-CC-2019/48,None,None,None +10233188161519ee,ANSSI-CC-2020/26,None,None,None +10233188161519ee,BSI-DSZ-CC-1059-2018,None,None,None +10233188161519ee,BSI-DSZ-CC-1059-V3-2019,None,None,None +102707a8c4d8a1b0,ANSSI-CC-2019/35,None,None,None +102707a8c4d8a1b0,ANSSI-CC-2020/26,None,None,None +102707a8c4d8a1b0,BSI-DSZ-CC-1059-2018,None,None,None +102707a8c4d8a1b0,BSI-DSZ-CC-1059-V3-2019,None,None,None +102bbdfcd696d7a9,ANSSI-CC-2018/13,None,None,None +102bbdfcd696d7a9,ANSSI-CC-2019/01,None,None,None +107c348f4ba0a41c,BSI-DSZ-CC-1135-2020,None,None,None +109eb2158ca6a2f9,BSI-DSZ-CC-0410-2007,None,None,None +10af4a6378d0fb7a,ANSSI-CC-2014/07,None,None,None +10af4a6378d0fb7a,ANSSI-CC-2014/86,None,None,None +10af4a6378d0fb7a,BSI-DSZ-CC-0837-V2-2014,None,None,None +10af4a6378d0fb7a,BSI-DSZ-CC-0845-V2-2013,None,None,None +10b17081dd7cad8f,ANSSI-CC-2015/36,None,None,None +10b17081dd7cad8f,ANSSI-CC-2017/39,None,None,None +112fe9f123e277ee,ANSSI-CC-2010/02,None,None,None +112fe9f123e277ee,ANSSI-CC-2010/19,None,None,None +114e39b4da949e6d,ANSSI-CC-2020/08,None,None,None +114e39b4da949e6d,BSI-DSZ-CC-1110-V2-2019,None,None,None +1172a88e009dc248,CRP257,None,None,None +1172a88e009dc248,CRP281,None,None,None +1188aebd8681e4fe,BSI-DSZ-CC-0465-2008,None,None,None +11a98c5d5ff6f40f,BSI-DSZ-CC-0403-2008,None,None,None +11a98c5d5ff6f40f,BSI-DSZ-CC-0468-2007,None,None,None +11a98c5d5ff6f40f,CRP244,None,None,None +122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None,None +123c94418ddf1ad8,BSI-DSZ-CC-0367-2006,None,None,None +131ec924c42ef90b,ANSSI-CC-2017/61,None,None,None +135ca3602f0005bf,BSI-DSZ-CC-0897-V2-2014,None,None,None +135ca3602f0005bf,NSCIB-CC-16-99111,None,None,None +13e5b88ad3becb77,BSI-DSZ-CC-0322-2005,None,None,None +13e5b88ad3becb77,BSI-DSZ-CC-0344-2005,None,None,None +1468948b4f0509ab,BSI-DSZ-CC-0813-2012,None,None,None +1494cff85b3dc2bb,BSI-DSZ-CC-0633-2010,None,None,None +157b741a0f529a0a,ANSSI-CC-2009/37,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0555-2009,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0633-2010,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0675-2011,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0745-2012,None,None,None +158950ed4bc35274,ANSSI-CC-2015/36,None,None,None +158950ed4bc35274,ANSSI-CC-2015/80,None,None,None +158950ed4bc35274,ANSSI-CC-2016/33,None,None,None +15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,None,None,None +15b121492722bffb,BSI-DSZ-CC-0754-2012,None,None,None +16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,None,None,None +16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,None,None,None +16b24ff1bf3c079b,ANSSI-CC-2012/22,None,None,None +173097995b7a7f12,ANSSI-CC-2012/49,None,None,None +173097995b7a7f12,ANSSI-CC-2013/28,None,None,None +173704f0d2b8a02f,ANSSI-CC-2018/08,None,None,None +173704f0d2b8a02f,ANSSI-CC-2018/19,None,None,None +177dee4c1051e612,BSI-DSZ-CC-0854-2013,None,None,None +17bfc3d0570ab3d0,ANSSI-CC-2016/09,None,None,None +183a5cd3e2aaa0ec,ANSSI-CC-2017/76,None,None,None +183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,None,None,None +18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,None,None,None +18b7e5bdf459ca13,NSCIB-CC-07-09219,None,None,None +195366bfba7213a5,BSI-DSZ-CC-0266-2005,None,None,None +195366bfba7213a5,BSI-DSZ-CC-0322-2005,None,None,None +195366bfba7213a5,BSI-DSZ-CC-0399-2007,None,None,None +1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,None,None,None +19d3185c40be008d,ANSSI-CC-2016/65,None,None,None +1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,None,None,None +1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,None,None,None +1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,None,None,None +1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,None,None,None +1cc05dbb992431b9,ANSSI-CC-2021/29,None,None,None +1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,None,None,None +1d9ae732c5dec242,ANSSI-CC-2009/37,None,None,None +1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,None,None,None +1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,None,None,None +1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,None,None,None +1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,None,None,None +2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,None,None,None +21951e191e55b66e,BSI-DSZ-CC-0782-2012,None,None,None +21cffadbbb87c205,BSI-DSZ-CC-0348-2006,None,None,None +22ede463dbf1a105,ANSSI-CC-2015/36,None,None,None +22ede463dbf1a105,ANSSI-CC-2015/80,None,None,None +22ede463dbf1a105,ANSSI-CC-2016/43,None,None,None +22ede463dbf1a105,ANSSI-CC-2016/44,None,None,None +23a85a7f9c07412f,ANSSI-CC-2014/46,None,None,None +23a85a7f9c07412f,ANSSI-CC-2016/33,None,None,None +24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,None,None,None +24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,None,None,None +259df98e7476a843,BSI-DSZ-CC-0666-2012,None,None,None +259e0ab1c4b690cf,ANSSI-CC-2012/70,None,None,None +25c07c5d0a05d86c,ANSSI-CC-2015/08,None,None,None +25c07c5d0a05d86c,ANSSI-CC-2015/15,None,None,None +25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,None,None,None +25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,None,None,None +26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,None,None,None +26c7b173668167d8,BSI-DSZ-CC-0813-2012,None,None,None +2760aeedce0b79db,ANSSI-CC-2011/64,None,None,None +2760aeedce0b79db,BSI-DSZ-CC-0555-2009,None,None,None +27ccec0740bb7915,ANSSI-CC-2010/02,None,None,None +27ccec0740bb7915,ANSSI-CC-2010/20,None,None,None +27ccec0740bb7915,ANSSI-CC-2010/35,None,None,None +296618f10b019d9c,BSI-DSZ-CC-0417-2008,None,None,None +29cc495628e76ffa,BSI-DSZ-CC-0845-2012,None,None,None +29cc495628e76ffa,BSI-DSZ-CC-0886-2013,None,None,None +29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,None,None,None +29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,None,None,None +2a07b85f61ac08b0,ANSSI-CC-2019/02,None,None,None +2a07b85f61ac08b0,ANSSI-CC-2019/59,None,None,None +2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,None,None,None +2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,None,None,None +2b0342d5b70b7a78,ANSSI-CC-2010/03,None,None,None +2bd9661657e78278,ANSSI-CC-2012/39,None,None,None +2bd9661657e78278,ANSSI-CC-2013/13,None,None,None +2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,None,None,None +2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,None,None,None +2d046178aa118fff,BSI-DSZ-CC-0293-2005,None,None,None +2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,None,None,None +2eee0fdd5cdaf565,ANSSI-CC-2017/07,None,None,None +2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,None,None,None +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,None,None,None +30c6777901ae346c,BSI-DSZ-CC-0438-2007,None,None,None +30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,None,None,None +3183669bf78db43b,ANSSI-CC-2015/45,None,None,None +3183669bf78db43b,ANSSI-CC-2016/64,None,None,None +3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,None,None,None +31fd07069aebf013,BSI-DSZ-CC-0466-2008,None,None,None +328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,None,None,None +328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,None,None,None +3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,None,None,None +3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,None,None,None +337ece90615ed69d,ANSSI-CC-2010/02,None,None,None +3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,None,None,None +33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,None,None,None +33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,None,None,None +34506b8fab6dd7c3,ANSSI-CC-2014/22,None,None,None +34ed76f0c32d9e93,ANSSI-CC-2012/49,None,None,None +34ed76f0c32d9e93,ANSSI-CC-2013/18,None,None,None +3519c14e4114d93d,BSI-DSZ-CC-0782-2012,None,None,None +35b9be2203d2b9e6,ANSSI-CC-2012/70,None,None,None +35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,None,None,None +35efe13fa9e93a68,CRP178,None,None,None +35efe13fa9e93a68,CRP182,None,None,None +36641ce7192b5f92,BSI-DSZ-CC-0977-2017,None,None,None +36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,None,None,None +373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,None,None,None +3746512864b941d4,ANSSI-CC-2016/15,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0404-2007,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0410-2007,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0555-2009,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0633-2010,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0674-2011,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0675-2011,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0709-2010,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0710-2010,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0730-2011,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0797-2012,None,None,None +37b94c08f8e25249,BSI-DSZ-CC-0799-2012,None,None,None +37c8d23c44b95833,ANSSI-CC-2017/16,None,None,None +37d28734245a3bb1,BSI-DSZ-CC-0257-2004,None,None,None +37e0d3cb098458d0,ANSSI-CC-2014/50,None,None,None +37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,None,None,None +37fe6036c2ac932b,ANSSI-CC-2014/06,None,None,None +37fe6036c2ac932b,ANSSI-CC-2014/07,None,None,None +37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,None,None,None +3817c0aca007989e,ANSSI-CC-2011/10,None,None,None +3817c0aca007989e,ANSSI-CC-2011/12,None,None,None +3817c0aca007989e,BSI-DSZ-CC-0523-2008,None,None,None +3817c0aca007989e,BSI-DSZ-CC-0626-2009,None,None,None +384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,None,None,None +384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,None,None,None +3976c9e492193315,BSI-DSZ-CC-1113-2021,None,None,None +39c07ebbca541145,BSI-DSZ-CC-0917-2014,None,None,None +3a3b0a9113835307,BSI-DSZ-CC-0402-2008,None,None,None +3a3b0a9113835307,BSI-DSZ-CC-0468-2007,None,None,None +3a3b0a9113835307,CCEVS-VR-07-0054,None,None,None +3a3b0a9113835307,CCEVS-VR-VID10271-2007,None,None,None +3a6a5f536bfbd4d4,ANSSI-CC-2009/59,None,None,None +3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,None,None,None +3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,None,None,None +3d149fe7c08bfc58,CRP256,None,None,None +3d756c83419bba28,ANSSI-CC-2011/07,None,None,None +3d756c83419bba28,ANSSI-CC-2011/77,None,None,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,None,None,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,None,None,None +3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,None,None,None +3e08a27e9d9c9b1e,NSCIB-CC-16-99111,None,None,None +3e182053b03f1faf,CCEVS-VR-06-0044,None,None,None +3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,None,None,None +3ea02a62856c752a,ANSSI-CC-2012/70,None,None,None +3ea02a62856c752a,ANSSI-CC-2012/72,None,None,None +3ea02a62856c752a,BSI-DSZ-CC-0719-2011,None,None,None +3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,None,None,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,None,None,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,None,None,None +3f4b6e4f245f6fab,ANSSI-CC-2017/49,None,None,None +3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,None,None,None +3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,None,None,None +3feda0b8b5637540,NSCIB-CC-67206-CR5,None,None,None +4057f5dbe4ffa4cd,ANSSI-CC-2017/57,None,None,None +40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,None,None,None +40fa0a4cb5193977,ANSSI-CC-2018/27,None,None,None +40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,None,None,None +40fc6ad0aed92913,ANSSI-CC-2017/54,None,None,None +40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,None,None,None +41bd2924e9afced5,ANSSI-CC-2010/49,None,None,None +41bd2924e9afced5,ANSSI-CC-2010/50,None,None,None +420876ec1e5ba657,ANSSI-CC-2012/30,None,None,None +424f6d496746258b,ANSSI-CC-2019/12,None,None,None +424f6d496746258b,BSI-DSZ-CC-0995-2018,None,None,None +4285d9b580f8a2a6,ANSSI-CC-2013/59,None,None,None +42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,None,None,None +42d2e68c29d9eef7,ANSSI-CC-2018/30,None,None,None +42f53e51476f1a3c,ANSSI-CC-2011/07,None,None,None +438b59086f6ebd64,BSI-DSZ-CC-1107-2020,None,None,None +43a4ca62d0c0b0da,ANSSI-CC-2017/63,None,None,None +43a4ca62d0c0b0da,ANSSI-CC-2017/64,None,None,None +43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,None,None,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,None,None,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,None,None,None +449c74a92ebb61a4,ANSSI-CC-2017/24,None,None,None +44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,None,None,None +45098872448f5816,NSCIB-CC-0441513-CR,None,None,None +46459cfb9c1045d8,ANSSI-CC-2018/27,None,None,None +46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,None,None,None +4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,None,None,None +47913a485c3c8a18,ANSSI-CC-2019/28,None,None,None +4832a44c0df0bad2,ANSSI-CC-2019/12,None,None,None +4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,None,None,None +4893140ae5daadaf,BSI-DSZ-CC-0782-2012,None,None,None +4893140ae5daadaf,CRP278,None,None,None +48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,None,None,None +48d897f17754c7ce,BSI-DSZ-CC-0978-2016,None,None,None +48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,None,None,None +491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,None,None,None +491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,None,None,None +49e6665ee56d3bfc,ANSSI-CC-2015/45,None,None,None +49e6665ee56d3bfc,ANSSI-CC-2016/64,None,None,None +49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,None,None,None +4abc83fee59586b2,ANSSI-CC-2020/07,None,None,None +4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,None,None,None +4abc83fee59586b2,NSCIB-CC-200270-CR,None,None,None +4b2f963b48e0f954,ANSSI-CC-2010/11,None,None,None +4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,None,None,None +4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,None,None,None +4bf7d6ef245dd393,ANSSI-CC-2009/60,None,None,None +4bf7d6ef245dd393,ANSSI-CC-2010/02,None,None,None +4c0e2794cb729825,ANSSI-CC-2010/02,None,None,None +4c0e2794cb729825,ANSSI-CC-2010/22,None,None,None +4c90a96fb1ee09f6,BSI-DSZ-CC-0410-2007,None,None,None +4d21561351ecd00a,BSI-DSZ-CC-1052-V3-2021,None,None,None +4d2a177384b23fd6,ANSSI-CC-2016/58,None,None,None +4d885e54f1361d04,BSI-DSZ-CC-0577-2009,None,None,None +4dbb108406acb1a5,ANSSI-CC-2010/02,None,None,None +4dbb108406acb1a5,ANSSI-CC-2012/68,None,None,None +4dc023ea2e3c4115,ANSSI-CC-2017/59,None,None,None +4dc023ea2e3c4115,BSI-DSZ-CC-0782-V3-2017,None,None,None +4dd0daec78093d31,ANSSI-CC-2016/59,None,None,None +4e29916c366fe997,BSI-DSZ-CC-0782-2012,None,None,None +4f3bf0c433b7f54f,BSI-DSZ-CC-0171-2002,None,None,None +4f8db13958be0f75,BSI-DSZ-CC-1051-2019,None,None,None +4f953ae00a24eb85,ANSSI-CC-2015/36,None,None,None +4f953ae00a24eb85,ANSSI-CC-2016/43,None,None,None +4f953ae00a24eb85,ANSSI-CC-2016/44,None,None,None +4f953ae00a24eb85,ANSSI-CC-2016/78,None,None,None +5103648897ba1b3a,BSI-DSZ-CC-0891-2015,None,None,None +5162627872566ada,ANSSI-CC-2020/07,None,None,None +5162627872566ada,BSI-DSZ-CC-1110-V2-2019,None,None,None +51768783ab3e67f3,ANSSI-CC-2010/03,None,None,None +51768783ab3e67f3,ANSSI-CC-2017/47,None,None,None +51768783ab3e67f3,ANSSI-CC-2017/49,None,None,None +51768783ab3e67f3,BSI-DSZ-CC-0955-V2-2016,None,None,None +51768783ab3e67f3,BSI-DSZ-CC-0973-V2-2016,None,None,None +51a4bea9a77a55cf,BSI-DSZ-CC-0410-2007,None,None,None +51a4bea9a77a55cf,BSI-DSZ-CC-0417-2008,None,None,None +51a4bea9a77a55cf,BSI-DSZ-CC-0602-2009,None,None,None +51c589803c349e44,BSI-DSZ-CC-0813-2012,None,None,None +51c589803c349e44,BSI-DSZ-CC-0851-2013,None,None,None +5346d5a4418466ab,BSI-DSZ-CC-1110-V3-2020,None,None,None +53bba35f7ea6edc9,ANSSI-CC-2017/02,None,None,None +53fe111411edfa45,BSI-DSZ-CC-0325-2006,None,None,None +541da571bac5e279,BSI-DSZ-CC-0466-2008,None,None,None +5423483cd3c3b75a,ANSSI-CC-2016/60,None,None,None +5423483cd3c3b75a,ANSSI-CC-2016/69,None,None,None +5423483cd3c3b75a,BSI-DSZ-CC-0946-V2-2015,None,None,None +5498bfcecf19dfa3,ANSSI-CC-2021/08,None,None,None +54f754dc95137c47,NSCIB-CC-13-37760-CR2,None,None,None +54fada9909a1edb3,ANSSI-CC-2009/56,None,None,None +54fada9909a1edb3,BSI-DSZ-CC-0451-2007,None,None,None +5506b5232076d409,CCEVS-VR-VID10334-2010,None,None,None +55669bcb3a9e09c0,BSI-DSZ-CC-1041-2018,None,None,None +55925bc39df20912,BSI-DSZ-CC-0322-2005,None,None,None +559e0907fef71cf6,ANSSI-CC-2014/62,None,None,None +559e0907fef71cf6,BSI-DSZ-CC-0829-2012,None,None,None +559e0907fef71cf6,BSI-DSZ-CC-0829-V2-2015,None,None,None +55e52bcaab69d573,BSI-DSZ-CC-0410-2007,None,None,None +55e52bcaab69d573,BSI-DSZ-CC-0463-2008,None,None,None +55e85aac6594c2fc,BSI-DSZ-CC-0410-2007,None,None,None +55e85aac6594c2fc,BSI-DSZ-CC-0417-2008,None,None,None +55e85aac6594c2fc,BSI-DSZ-CC-0643-2010,None,None,None +55f9f2fedfc7f069,BSI-DSZ-CC-0926-2014,None,None,None +568c473e6e5e95c7,BSI-DSZ-CC-0897-V2-2014,None,None,None +56a56d39dbfe3516,ANSSI-CC-2017/47,None,None,None +56a56d39dbfe3516,ANSSI-CC-2017/49,None,None,None +56a56d39dbfe3516,BSI-DSZ-CC-0973-V2-2016,None,None,None +56bca69ce012d4c1,BSI-DSZ-CC-0951-2015,None,None,None +56bca69ce012d4c1,BSI-DSZ-CC-1028-2017,None,None,None +56ceb1a9c003ac3f,BSI-DSZ-CC-1124-2021,None,None,None +56ebfd7a918891c6,2016-36-INF-1779,None,None,None +5787ff8207fd751d,ANSSI-CC-2021/51,None,None,None +57e108dcd6ee072b,BSI-DSZ-CC-0720-2011,None,None,None +57e929939f1fd1c0,BSI-DSZ-CC-0487-2009,None,None,None +583c12b5bf2423c6,BSI-DSZ-CC-1110-V4-2021,None,None,None +58688dfc2c59d3ab,BSI-DSZ-CC-0831-V2-2020,None,None,None +58688dfc2c59d3ab,BSI-DSZ-CC-0957-V2-2016,None,None,None +587055293e9fa51d,ANSSI-CC-2017/07,None,None,None +587055293e9fa51d,BSI-DSZ-CC-0891-V2-2016,None,None,None +58ece75f7f2b5099,ANSSI-CC-2010/02,None,None,None +58ece75f7f2b5099,ANSSI-CC-2012/68,None,None,None +59820117bb7fa240,ANSSI-CC-2009/61,None,None,None +59820117bb7fa240,ANSSI-CC-2010/02,None,None,None +59820117bb7fa240,ANSSI-CC-2010/31,None,None,None +598e2a9978d3c79c,ANSSI-CC-2013/59,None,None,None +59b026d889aa0309,BSI-DSZ-CC-1025-V2-2019,None,None,None +5a1764ef77908457,BSI-DSZ-CC-1040-2019,None,None,None +5b1b55f1997a9439,OCSI/CERT/IMQ/04/2016/RC,None,None,None +5b67a83de2aaa93a,ANSSI-CC-2013/02,None,None,None +5b67a83de2aaa93a,ANSSI-CC-2014/37,None,None,None +5bf118ffa29520c0,ANSSI-CC-2017/76,None,None,None +5bf118ffa29520c0,BSI-DSZ-CC-0891-V2-2016,None,None,None +5bfbc4a6cbab9eb1,BSI-DSZ-CC-0302-2005,None,None,None +5c5431cca026fb3e,BSI-DSZ-CC-0179-2003,None,None,None +5c5431cca026fb3e,BSI-DSZ-CC-0213-2003,None,None,None +5caf1c90ec7001b7,BSI-DSZ-CC-0410-2007,None,None,None +5caf1c90ec7001b7,BSI-DSZ-CC-0417-2008,None,None,None +5cbd46653c3bea1a,BSI-DSZ-CC-0439-2008,None,None,None +5cec11774890d61e,BSI-DSZ-CC-0750-V2-2014,None,None,None +5cec11774890d61e,BSI-DSZ-CC-0858-2013,None,None,None +5cec11774890d61e,BSI-DSZ-CC-0911-2014,None,None,None +5cec11774890d61e,BSI-DSZ-CC-0912-2014,None,None,None +5cec11774890d61e,BSI-DSZ-CC-0914-2014,None,None,None +5cec11774890d61e,NSCIB-CC-13-37760-CR2,None,None,None +5cfe68ed449e0478,ANSSI-CC-2017/76,None,None,None +5cfe68ed449e0478,BSI-DSZ-CC-0891-V2-2016,None,None,None +5d4aae047020163b,ANSSI-CC-2010/03,None,None,None +5d4aae047020163b,ANSSI-CC-2017/47,None,None,None +5d4aae047020163b,ANSSI-CC-2017/49,None,None,None +5d4aae047020163b,BSI-DSZ-CC-0955-V2-2016,None,None,None +5d4aae047020163b,BSI-DSZ-CC-0973-V2-2016,None,None,None +5d674c06a559a5e7,BSI-DSZ-CC-1128-V4-2022,None,None,None +5e4e16da9d0630dd,ANSSI-CC-2016/70,None,None,None +5e4e16da9d0630dd,ANSSI-CC-2017/59,None,None,None +5eb91b620bbf5bfc,ANSSI-CC-2009/39,None,None,None +5eb91b620bbf5bfc,ANSSI-CC-2009/40,None,None,None +5eda77c801e2df86,BSI-DSZ-CC-0891-V4-2019,None,None,None +5edefd19948ad581,CRP237,None,None,None +5ee9898421c06859,BSI-DSZ-CC-0758-2012,None,None,None +5efe98a1ba4df4d7,BSI-DSZ-CC-0757-2011,None,None,None +5f259ce91f0234a0,BSI-DSZ-CC-1110-V3-2020,None,None,None +5f4cd1504f8b18e3,CCEVS-VR-05-0131,None,None,None +5f84cd800ee96468,BSI-DSZ-CC-0750-V2-2014,None,None,None +5f84cd800ee96468,NSCIB-CC-13-37760-CR2,None,None,None +5f99b00bf258f32f,ANSSI-CC-2010/03,None,None,None +5f99b00bf258f32f,ANSSI-CC-2017/47,None,None,None +5f99b00bf258f32f,ANSSI-CC-2017/49,None,None,None +5f99b00bf258f32f,BSI-DSZ-CC-0955-V2-2016,None,None,None +5f99b00bf258f32f,BSI-DSZ-CC-0973-V2-2016,None,None,None +60ec8738947badb0,ANSSI-CC-2017/12,None,None,None +60ec8738947badb0,ANSSI-CC-2018/12,None,None,None +60f08c85a518de1c,ANSSI-CC-2009/61,None,None,None +60f08c85a518de1c,ANSSI-CC-2010/02,None,None,None +60f0dd83c8f32b8c,ANSSI-CC-2014/46,None,None,None +6136acacc653106c,BSI-DSZ-CC-0555-2009,None,None,None +61e862ed07d3cb03,BSI-DSZ-CC-0875-2015,None,None,None +62313881916550ca,ANSSI-CC-2019/28,None,None,None +62eb991153211de6,ANSSI-CC-2010/02,None,None,None +62eb991153211de6,BSI-DSZ-CC-0671-2011,None,None,None +6313a92bf426d2bb,BSI-DSZ-CC-0411-2007,None,None,None +6337d3dcc5362dc6,ANSSI-CC-2019/12,None,None,None +637657613004dbbd,BSI-DSZ-CC-0293-2005,None,None,None +63e55780fa1766de,ANSSI-CC-2010/01,None,None,None +64e80c255d51ff37,BSI-DSZ-CC-0404-2007,None,None,None +64e80c255d51ff37,BSI-DSZ-CC-0410-2007,None,None,None +64e80c255d51ff37,BSI-DSZ-CC-0411-2007,None,None,None +64ed7e55bdc50177,ANSSI-CC-2020/07,None,None,None +64ed7e55bdc50177,BSI-DSZ-CC-1110-V2-2019,None,None,None +6529877d9799822a,BSI-DSZ-CC-1113-V3-2022,None,None,None +654554952ecb20ac,ANSSI-CC-2010/01,None,None,None +6587b26878da24b5,ANSSI-CC-2015/08,None,None,None +6587b26878da24b5,ANSSI-CC-2015/15,None,None,None +6587b26878da24b5,ANSSI-CC-2015/30,None,None,None +6587b26878da24b5,BSI-DSZ-CC-0845-2012,None,None,None +6587b26878da24b5,BSI-DSZ-CC-0870-2014,None,None,None +65a47f93451f13de,BSI-DSZ-CC-1079-V2-2020,None,None,None +65af80e5c6538215,BSI-DSZ-CC-1059-V2-2019,None,None,None +65fba51313f65938,ANSSI-CC-2015/66,None,None,None +6700028a309c3f7f,CRP271,None,None,None +6700028a309c3f7f,CRP282,None,None,None +67240adc18be338d,BSI-DSZ-CC-1044-V5-2022,None,None,None +67e75c6c6538110e,BSI-DSZ-CC-0963-2015,None,None,None +68875c05e00bd1aa,ANSSI-CC-2009/26,None,None,None +692e91451741ef49,BSI-DSZ-CC-0268-2005,None,None,None +693325ca20a6325b,BSI-DSZ-CC-0917-2014,None,None,None +696c193b3d13924e,OCSI/CERT/RES/03/2012/RC,None,None,None +6999db84bd629b32,BSI-DSZ-CC-0293-2005,None,None,None +6999db84bd629b32,BSI-DSZ-CC-0296-2006,None,None,None +69b877d99c360f97,BSI-DSZ-CC-0837-V2-2014,None,None,None +6a46ff1d1a5154b0,BSI-DSZ-CC-0845-2012,None,None,None +6abef2102c86c0fc,ANSSI-CC-2011/07,None,None,None +6ae356d8abbfdeb9,BSI-DSZ-CC-1153-2021,None,None,None +6b605ff743c859d0,ANSSI-CC-2015/36,None,None,None +6b605ff743c859d0,ANSSI-CC-2017/02,None,None,None +6b605ff743c859d0,ANSSI-CC-2018/41,None,None,None +6bdca9f13e19572e,ANSSI-CC-2019/34,None,None,None +6bdca9f13e19572e,ANSSI-CC-2020/26,None,None,None +6bdca9f13e19572e,BSI-DSZ-CC-1059-2018,None,None,None +6bdca9f13e19572e,BSI-DSZ-CC-1059-V3-2019,None,None,None +6c2fc8308efeccb4,BSI-DSZ-CC-0555-2009,None,None,None +6c2fc8308efeccb4,BSI-DSZ-CC-0633-2010,None,None,None +6c2fc8308efeccb4,BSI-DSZ-CC-0675-2011,None,None,None +6c3e0fe95400f1f5,BSI-DSZ-CC-0939-V2-2016,None,None,None +6cef8442092da1da,ANSSI-CC-2021/52,None,None,None +6d9c2bab8cc8d28c,ANSSI-CC-2014/06,None,None,None +6d9c2bab8cc8d28c,ANSSI-CC-2014/14,None,None,None +6d9c2bab8cc8d28c,ANSSI-CC-2014/49,None,None,None +6d9c2bab8cc8d28c,BSI-DSZ-CC-0695-2011,None,None,None +6d9c2bab8cc8d28c,BSI-DSZ-CC-0829-2012,None,None,None +6dde54b8c6d592ef,BSI-DSZ-CC-1086-2018,None,None,None +6df8f86449e56c05,ANSSI-CC-2017/24,None,None,None +6df8f86449e56c05,ANSSI-CC-2018/32,None,None,None +6e9496b7254e02e2,ANSSI-CC-2020/08,None,None,None +6e9496b7254e02e2,BSI-DSZ-CC-1110-V2-2019,None,None,None +6eacef3b2f976ce9,BSI-DSZ-CC-0814-2012,None,None,None +6eeed6fefb1f0243,ANSSI-CC-2019/28,None,None,None +6ef82dead8e3437f,BSI-DSZ-CC-0837-2013,None,None,None +6ef82dead8e3437f,BSI-DSZ-CC-0892-2014,None,None,None +6ef82dead8e3437f,NSCIB-CC-12-36243,None,None,None +6f13e8b708b43383,ANSSI-CC-2009/28,None,None,None +6f480809fc73b635,BSI-DSZ-CC-0978-V2-2017,None,None,None +6f90b0bf6bbe884b,BSI-DSZ-CC-0916-2015,None,None,None +6f90b0bf6bbe884b,BSI-DSZ-CC-0950-2017,None,None,None +6fd73557577cf9ed,ANSSI-CC-2020/21,None,None,None +6fe5828936663aed,BSI-DSZ-CC-0482-2008,None,None,None +6ff89f3123a6a98f,BSI-DSZ-CC-0782-V3-2017,None,None,None +70908a4cfdc5585c,BSI-DSZ-CC-0266-2005,None,None,None +70908a4cfdc5585c,BSI-DSZ-CC-0437-2008,None,None,None +7096b882b315807a,BSI-DSZ-CC-0955-V2-2016,None,None,None +70f75a14390522b6,BSI-DSZ-CC-0837-V2-2014,None,None,None +711a7b593ed41b02,ANSSI-CC-2010/49,None,None,None +71d3720cad53b289,BSI-DSZ-CC-0891-V4-2019,None,None,None +71d3720cad53b289,BSI-DSZ-CC-1105-2020,None,None,None +71eae9be60b7fb6c,BSI-DSZ-CC-0555-2009,None,None,None +71eae9be60b7fb6c,BSI-DSZ-CC-0633-2010,None,None,None +723ba544431f7874,BSI-DSZ-CC-1110-V4-2021,None,None,None +731aac44217611ec,ANSSI-CC-2014/24,None,None,None +736ede0289146b59,BSI-DSZ-CC-1067-2020,None,None,None +7386d04b20f354f4,BSI-DSZ-CC-0829-2012,None,None,None +7386d04b20f354f4,BSI-DSZ-CC-0829-V2-2015,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0404-2007,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0410-2007,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0555-2009,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0633-2010,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0674-2011,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0675-2011,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0709-2010,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0710-2010,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0730-2011,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0797-2012,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0798-2012,None,None,None +743318f3a2b4e281,BSI-DSZ-CC-0799-2012,None,None,None +74d685c3723bdf92,BSI-DSZ-CC-0978-2016,None,None,None +75ce95f62fc65fac,BSI-DSZ-CC-0269-2006,None,None,None +75ce95f62fc65fac,BSI-DSZ-CC-0376-2006,None,None,None +76d52f916b4222b1,BSI-DSZ-CC-0266-2005,None,None,None +76d52f916b4222b1,BSI-DSZ-CC-0470-2008,None,None,None +781c243d0021d4c5,BSI-DSZ-CC-0223-2003,None,None,None +78a279b07a6d3367,CCEVS-VR-05-0119,None,None,None +78ef46252d580af0,ANSSI-CC-2016/22,None,None,None +793075287bf70a81,BSI-DSZ-CC-1139-V2-2021,None,None,None +7942e835fb18c4ac,BSI-DSZ-CC-0482-2008,None,None,None +797ea41cc2e95f08,ANSSI-CC-2012/22,None,None,None +797ea41cc2e95f08,ANSSI-CC-2013/36,None,None,None +7980d204dbc3ecee,BSI-DSZ-CC-1113-2021,None,None,None +7a059d2754e088a3,ANSSI-CC-2022/10,None,None,None +7a059d2754e088a3,ANSSI-CC-2022/13,None,None,None +7a059d2754e088a3,BSI-DSZ-CC-1107-V2-2021,None,None,None +7a26d92281df3e8e,BSI-DSZ-CC-0451-2007,None,None,None +7a5a4a9578503a55,BSI-DSZ-CC-1072-V3-2019,None,None,None +7b63c31cfd0eb65f,BSI-DSZ-CC-0977-V2-2019,None,None,None +7b63c31cfd0eb65f,BSI-DSZ-CC-1019-V2-2019,None,None,None +7c04881c9a46b337,NSCIB-CC-09-11025,None,None,None +7c708f62e3280ec3,BSI-DSZ-CC-0354-2006,None,None,None +7c7e48e3aed8bfd4,ANSSI-CC-2020/08,None,None,None +7c7e48e3aed8bfd4,BSI-DSZ-CC-1110-V2-2019,None,None,None +7d4585a4b5b6e873,ANSSI-CC-2010/02,None,None,None +7dfcefd1ffeaa941,ANSSI-CC-2018/12,None,None,None +7e24168630d5124e,ANSSI-CC-2021/02,None,None,None +7e37ff763a97bc72,BSI-DSZ-CC-0555-2009,None,None,None +7e37ff763a97bc72,BSI-DSZ-CC-0857-2013,None,None,None +7e8763f553e43d12,BSI-DSZ-CC-0782-2012,None,None,None +7e92dcac20e009a5,BSI-DSZ-CC-0827-V3-2015,None,None,None +7f02a580f5219377,BSI-DSZ-CC-1040-2019,None,None,None +8035f242b12cec6d,ANSSI-CC-2009/26,None,None,None +8035f242b12cec6d,ANSSI-CC-2009/28,None,None,None +8035f242b12cec6d,ANSSI-CC-2009/50,None,None,None +808de32aa819821d,BSI-DSZ-CC-1021-V2-2017,None,None,None +80fe3ba3d9c9bf64,BSI-DSZ-CC-0406-2007,None,None,None +80fe3ba3d9c9bf64,BSI-DSZ-CC-0410-2007,None,None,None +819d92a6492415d3,BSI-DSZ-CC-0728-2011,None,None,None +819d92a6492415d3,BSI-DSZ-CC-0763-2012,None,None,None +82a2bd8fe4ed2087,BSI-DSZ-CC-0645-2010,None,None,None +82a2bd8fe4ed2087,BSI-DSZ-CC-0731-2011,None,None,None +82a2bd8fe4ed2087,BSI-DSZ-CC-0743-2011,None,None,None +82a2bd8fe4ed2087,BSI-DSZ-CC-0750-2011,None,None,None +83020f91ce1aaf68,ANSSI-CC-2017/24,None,None,None +83020f91ce1aaf68,ANSSI-CC-2018/32,None,None,None +835211383d281d89,BSI-DSZ-CC-1079-V2-2020,None,None,None +83b4d7d35a747d1b,BSI-DSZ-CC-0829-2012,None,None,None +83bb8420c3cccf85,BSI-DSZ-CC-0464-2008,None,None,None +83c1ee8ab43ea00a,ANSSI-CC-2010/02,None,None,None +84444ea4fabcc66c,BSI-DSZ-CC-1107-V2-2021,None,None,None +84501b4439b66fd9,BSI-DSZ-CC-0837-V2-2014,None,None,None +84b1520706c3d784,NSCIB-CC-13-37760-CR2,None,None,None +84b1520706c3d784,OCSI/CERT/SYS/05/2015/RC,None,None,None +84c8bf1f809380bc,ANSSI-CC-2018/12,None,None,None +84ff17001564ea47,BSI-DSZ-CC-1040-2019,None,None,None +85fa17a19251a2ea,BSI-DSZ-CC-0410-2007,None,None,None +85fa17a19251a2ea,BSI-DSZ-CC-0709-2010,None,None,None +86462bcec4492bfa,ANSSI-CC-2017/73,None,None,None +86462bcec4492bfa,ANSSI-CC-2020/22,None,None,None +86e9da10bc176267,ANSSI-CC-2020/05,None,None,None +871066e9e2eaba52,ANSSI-CC-2022/39,None,None,None +8782c7c292ef2759,BSI-DSZ-CC-0782-V2-2015,None,None,None +8782c7c292ef2759,BSI-DSZ-CC-0879-V2-2015,None,None,None +878af76320373ba0,BSI-DSZ-CC-0874-2014,None,None,None +87abd998f3b7e26f,ANSSI-CC-2015/36,None,None,None +87abd998f3b7e26f,ANSSI-CC-2016/43,None,None,None +87abd998f3b7e26f,ANSSI-CC-2016/44,None,None,None +87abd998f3b7e26f,ANSSI-CC-2016/77,None,None,None +8817adf6eaf4f06c,OCSI/CERT/SYS/10/2016/RC,None,None,None +8820852f4a1163b3,BSI-DSZ-CC-1110-V2-2019,None,None,None +8825e2c99223c084,BSI-DSZ-CC-0427-2007,None,None,None +882a9d016c7e1f36,ANSSI-CC-2009/38,None,None,None +8865aed38c79d3fc,BSI-DSZ-CC-0606-2010,None,None,None +8870ddf70bf57890,ANSSI-CC-2012/70,None,None,None +8870ddf70bf57890,ANSSI-CC-2012/72,None,None,None +8944c4f14bbcaa74,ANSSI-CC-2020/94,None,None,None +89d1f7ab8b77e41d,BSI-DSZ-CC-0831-2018,None,None,None +89d1f7ab8b77e41d,BSI-DSZ-CC-0957-V2-2016,None,None,None +8a5f8686ab9de19e,BSI-DSZ-CC-0891-2015,None,None,None +8a7731e9eb363fb4,BSI-DSZ-CC-0410-2007,None,None,None +8a7731e9eb363fb4,BSI-DSZ-CC-0417-2008,None,None,None +8ade0910a89717a8,BSI-DSZ-CC-0977-2017,None,None,None +8ae9a7fbbb50b6ed,ANSSI-CC-2012/77,None,None,None +8b0204e041dd3eb8,ANSSI-CC-2019/12,None,None,None +8b7145c13f36c324,BSI-DSZ-CC-1110-V3-2020,None,None,None +8bbbd89d09bb82ab,BSI-DSZ-CC-0555-2009,None,None,None +8bbbd89d09bb82ab,BSI-DSZ-CC-0857-2013,None,None,None +8bc3658bc958f81f,BSI-DSZ-CC-0837-V2-2014,None,None,None +8c6c4b4606608dfe,ANSSI-CC-2012/08,None,None,None +8cfcf8dc64485a74,ANSSI-CC-2009/28,None,None,None +8cfd0c9f4bcd21b8,BSI-DSZ-CC-0707-2012,None,None,None +8cfd0c9f4bcd21b8,BSI-DSZ-CC-0864-2012,None,None,None +8d2c50e41a1a0061,ANSSI-CC-2012/77,None,None,None +8d68ee5c283ca6b0,BSI-DSZ-CC-0552-2009,None,None,None +8d92202f35f15712,ANSSI-CC-2013/27,None,None,None +8d92202f35f15712,ANSSI-CC-2014/56,None,None,None +8dcdbce3262bfcd6,ANSSI-CC-2014/58,None,None,None +8e713eb632acc712,BSI-DSZ-CC-0782-V4-2018,None,None,None +8ee7145a1b48b578,ANSSI-CC-2017/76,None,None,None +8ee7145a1b48b578,BSI-DSZ-CC-0891-V4-2019,None,None,None +8f7cc0f456186028,BSI-DSZ-CC-0203-2003,None,None,None +8f7cc0f456186028,BSI-DSZ-CC-0205-2003,None,None,None +8f7e22207cb3dd8b,ANSSI-CC-2014/06,None,None,None +8f7e22207cb3dd8b,BSI-DSZ-CC-0829-2012,None,None,None +8f81bb35e37f1b4c,ANSSI-CC-2021/29,None,None,None +8f81bb35e37f1b4c,BSI-DSZ-CC-1107-2020,None,None,None +902dbd430adb77a5,BSI-DSZ-CC-0869-V2-2019,None,None,None +902dbd430adb77a5,BSI-DSZ-CC-0891-V3-2018,None,None,None +915e919390e6f58b,ANSSI-CC-2015/59,None,None,None +921ecc7c3e5f4884,BSI-DSZ-CC-0309-2006,None,None,None +9231adbf747903df,ANSSI-CC-2012/27,None,None,None +925da3b468003e72,ANSSI-CC-2021/35-R01,None,None,None +925da3b468003e72,NSCIB-CC-0530096-CR,None,None,None +9292ab2134e8a712,BSI-DSZ-CC-0827-V7-2018,None,None,None +9292ab2134e8a712,BSI-DSZ-CC-1015-2017,None,None,None +9299f6cd8a266541,BSI-DSZ-CC-0404-2007,None,None,None +9299f6cd8a266541,BSI-DSZ-CC-0417-2008,None,None,None +92ae986997c1d45c,BSI-DSZ-CC-0977-2017,None,None,None +92ae986997c1d45c,BSI-DSZ-CC-1019-2017,None,None,None +92c7355f5ea28e52,CRP225,None,None,None +932cef2b9698dde8,BSI-DSZ-CC-0283-2006,None,None,None +9337ab9573b366c1,BSI-DSZ-CC-0329-2006,None,None,None +93f14cad1cb201fa,BSI-DSZ-CC-0827-V6-2017,None,None,None +940a18f62c79ffae,ANSSI-CC-2021/51,None,None,None +953e132dd43ec03f,BSI-DSZ-CC-1058-2018,None,None,None +9545f2b0b8b92c8d,BSI-DSZ-CC-0410-2007,None,None,None +9545f2b0b8b92c8d,BSI-DSZ-CC-0527-2008,None,None,None +95774db1eb19b767,ANSSI-CC-2018/27,None,None,None +95c52fa37ef1eb3b,BSI-DSZ-CC-0791-2012,None,None,None +96106c2d31eea8b2,BSI-DSZ-CC-1039-2017,None,None,None +966a03f9f111aa35,ANSSI-CC-2021/42,None,None,None +966a03f9f111aa35,BSI-DSZ-CC-1107-V2-2021,None,None,None +96b0895b9f712d2f,BSI-DSZ-CC-0410-2007,None,None,None +96b0895b9f712d2f,BSI-DSZ-CC-0518-2008,None,None,None +96c685a3cdd9ed59,BSI-DSZ-CC-0510-2008,None,None,None +96c904c81aa2a241,ANSSI-CC-2017/07,None,None,None +96c904c81aa2a241,BSI-DSZ-CC-0891-V2-2016,None,None,None +975dae6e02765a90,ANSSI-CC-2009/63,None,None,None +977b04d04def60a7,ANSSI-CC-2019/28,None,None,None +97eb8171b51f9aad,BSI-DSZ-CC-0944-2014,None,None,None +97eb8171b51f9aad,BSI-DSZ-CC-0963-V2-2017,None,None,None +97f302708066dc04,BSI-DSZ-CC-0639-2010,None,None,None +981f7ea91d7bdcb6,ANSSI-CC-2014/06,None,None,None +981f7ea91d7bdcb6,BSI-DSZ-CC-0837-2013,None,None,None +983d431922822165,BSI-DSZ-CC-0837-2013,None,None,None +98642c6f99438194,ANSSI-CC-2020/04,None,None,None +988394e53c8e52d9,BSI-DSZ-CC-0976-V3-2019,None,None,None +988394e53c8e52d9,BSI-DSZ-CC-1110-V3-2020,None,None,None +98a578f7758bb518,BSI-DSZ-CC-0438-2007,None,None,None +9938be53082a02e6,NSCIB-CC-16-99111,None,None,None +994a2bf994b4facd,ANSSI-CC-2010/02,None,None,None +994a2bf994b4facd,ANSSI-CC-2012/68,None,None,None +994a2bf994b4facd,ANSSI-CC-2015/16,None,None,None +996691f0e03e303d,BSI-DSZ-CC-0555-2009,None,None,None +997281c99ac93b3b,ANSSI-CC-2010/03,None,None,None +997281c99ac93b3b,ANSSI-CC-2010/04,None,None,None +9973eb31c46f8637,ANSSI-CC-2010/08,None,None,None +9973eb31c46f8637,ANSSI-CC-2011/67,None,None,None +9ac6a8bda4fb7dc4,BSI-DSZ-CC-0751-2013,None,None,None +9ac6a8bda4fb7dc4,BSI-DSZ-CC-0813-2012,None,None,None +9ae7edbd0444982d,BSI-DSZ-CC-0837-V2-2014,None,None,None +9b30bcdbccfd4c68,BSI-DSZ-CC-0740-2011,None,None,None +9ba1fa1ae8c3ae67,ANSSI-CC-2016/65,None,None,None +9be0c86102117436,ANSSI-CC-2021/35-R01,None,None,None +9be76c10474e0c80,BSI-DSZ-CC-0758-2012,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0404-2007,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0410-2007,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0555-2009,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0633-2010,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0674-2011,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0675-2011,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0680-2010,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0709-2010,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0710-2010,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0730-2011,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0798-2012,None,None,None +9cc6c08cbaa694f2,BSI-DSZ-CC-0799-2012,None,None,None +9d36f8ab4e1b45dc,BSI-DSZ-CC-0324-2006,None,None,None +9e55246befe96fb4,ANSSI-CC-2010/01,None,None,None +9e7c513fdd35df17,ANSSI-CC-2010/40,None,None,None +9e7c513fdd35df17,ANSSI-CC-2011/64,None,None,None +9e7c513fdd35df17,BSI-DSZ-CC-0555-2009,None,None,None +9e7c513fdd35df17,BSI-DSZ-CC-0645-2010,None,None,None +9f400f89490fb8d5,BSI-DSZ-CC-0813-2012,None,None,None +9f8c9a3aa64a9f8e,ANSSI-CC-2019/01,None,None,None +a00e93f2981ef418,CRP292,None,None,None +a0fff0700364df49,BSI-DSZ-CC-0829-V2-2015,None,None,None +a0fff0700364df49,BSI-DSZ-CC-0952-2014,None,None,None +a14f221cf1761a8c,ANSSI-CC-2015/15,None,None,None +a14f221cf1761a8c,BSI-DSZ-CC-0845-V2-2013,None,None,None +a191b2da409f3518,BSI-DSZ-CC-0410-2007,None,None,None +a191b2da409f3518,BSI-DSZ-CC-0521-2008,None,None,None +a289b9d43310182b,BSI-DSZ-CC-0973-2016,None,None,None +a289b9d43310182b,BSI-DSZ-CC-0973-V2-2016,None,None,None +a29d46ab6efee468,BSI-DSZ-CC-0837-V2-2014,None,None,None +a2a5ccdf0846ce67,BSI-DSZ-CC-0460-2008,None,None,None +a2cbd066271b6b3b,BSI-DSZ-CC-0586-2009,None,None,None +a334142059d865a2,ANSSI-CC-2017/75,None,None,None +a334142059d865a2,ANSSI-CC-2018/61,None,None,None +a35ed74b7ea3ec4b,ANSSI-CC-2012/68,None,None,None +a3d152e25f8ce6c9,BSI-DSZ-CC-0870-2014,None,None,None +a465a767ceb7fd30,BSI-DSZ-CC-1015-2017,None,None,None +a49706f436a6873d,ANSSI-CC-2012/22,None,None,None +a49706f436a6873d,ANSSI-CC-2013/36,None,None,None +a5451ba38050a44d,BSI-DSZ-CC-0400-2007,None,None,None +a57845715134651a,BSI-DSZ-CC-1040-2019,None,None,None +a5898123ec124b34,ANSSI-CC-2015/36,None,None,None +a5898123ec124b34,ANSSI-CC-2015/80,None,None,None +a5898123ec124b34,ANSSI-CC-2016/43,None,None,None +a5898123ec124b34,ANSSI-CC-2016/44,None,None,None +a596f2412c0f8fe2,BSI-DSZ-CC-0808-2013,None,None,None +a596f2412c0f8fe2,BSI-DSZ-CC-0845-2012,None,None,None +a596f2412c0f8fe2,BSI-DSZ-CC-0885-2013,None,None,None +a5e8267322d0d7a6,ANSSI-CC-2010/40,None,None,None +a5e8267322d0d7a6,ANSSI-CC-2011/64,None,None,None +a5e8267322d0d7a6,BSI-DSZ-CC-0555-2009,None,None,None +a5e8267322d0d7a6,BSI-DSZ-CC-0645-2010,None,None,None +a64e6bf63ab6a299,ANSSI-CC-2013/32,None,None,None +a64e6bf63ab6a299,BSI-DSZ-CC-0829-2012,None,None,None +a67919286833a7df,BSI-DSZ-CC-0918-V2-2021,None,None,None +a67919286833a7df,BSI-DSZ-CC-0957-V2-2016,None,None,None +a6c1dd30884a197c,ANSSI-CC-2010/08,None,None,None +a76054a284f0ce61,ANSSI-CC-2014/86,None,None,None +a76054a284f0ce61,BSI-DSZ-CC-0837-V2-2014,None,None,None +a781ac2579798523,BSI-DSZ-CC-1059-V4-2021,None,None,None +a851c3b4377f4923,ANSSI-CC-2011/07,None,None,None +a9004216663480de,BSI-DSZ-CC-0782-V2-2015,None,None,None +a925d8f1adc5cfe2,BSI-DSZ-CC-0891-V3-2018,None,None,None +a925d8f1adc5cfe2,BSI-DSZ-CC-1001-2018,None,None,None +a925d8f1adc5cfe2,BSI-DSZ-CC-1033-2019,None,None,None +a925d8f1adc5cfe2,BSI-DSZ-CC-1034-2019,None,None,None +a93edea77d9f3338,BSI-DSZ-CC-0410-2007,None,None,None +a9f542d8e2be69fd,BSI-DSZ-CC-0782-2012,None,None,None +aa0213dd9f727219,BSI-DSZ-CC-0782-2012,None,None,None +aa0213dd9f727219,BSI-DSZ-CC-0879-2014,None,None,None +aa3f466daa34d3ff,BSI-DSZ-CC-0377-2007,None,None,None +aadbb9992a672a1b,ANSSI-CC-2019/01,None,None,None +ab769495f97aab9c,BSI-DSZ-CC-0879-V3-2018,None,None,None +ab88698a0c6ee1bb,BSI-DSZ-CC-0640-2010,None,None,None +ab8cf6af80a5c0e0,ANSSI-CC-2017/49,None,None,None +abef2b316c54f9e7,BSI-DSZ-CC-0978-2016,None,None,None +ac1257069b2f4afd,BSI-DSZ-CC-1113-V2-2021,None,None,None +ac4687f4b15507b6,ANSSI-CC-2014/58,None,None,None +ac4687f4b15507b6,ANSSI-CC-2014/59,None,None,None +acee5f619f09bf79,2015-27-INF-1539,None,None,None +ad2351479ff6ee84,ANSSI-CC-2009/61,None,None,None +ad2351479ff6ee84,ANSSI-CC-2010/02,None,None,None +ad2351479ff6ee84,ANSSI-CC-2010/29,None,None,None +adb678f7143014a0,ANSSI-CC-2015/15,None,None,None +adb678f7143014a0,BSI-DSZ-CC-0845-V2-2013,None,None,None +adc6971cdb0613b0,ANSSI-CC-2011/10,None,None,None +adc6971cdb0613b0,BSI-DSZ-CC-0523-2008,None,None,None +adc6971cdb0613b0,BSI-DSZ-CC-0626-2009,None,None,None +adc9df2489ce090a,BSI-DSZ-CC-0891-V4-2019,None,None,None +adc9df2489ce090a,BSI-DSZ-CC-1033-2019,None,None,None +adc9df2489ce090a,BSI-DSZ-CC-1034-V2-2021,None,None,None +adc9df2489ce090a,BSI-DSZ-CC-1105-2020,None,None,None +ae175aa839dbf692,NSCIB-CC-217812-CR3,None,None,None +ae9574428bd713e6,BSI-DSZ-CC-0891-V3-2018,None,None,None +ae9574428bd713e6,BSI-DSZ-CC-0995-2018,None,None,None +ae9574428bd713e6,BSI-DSZ-CC-1034-2019,None,None,None +aeb79c7e573de0e5,ANSSI-CC-2014/14,None,None,None +aeb79c7e573de0e5,ANSSI-CC-2014/86,None,None,None +aeb79c7e573de0e5,BSI-DSZ-CC-0837-2013,None,None,None +aeb79c7e573de0e5,BSI-DSZ-CC-0837-V2-2014,None,None,None +aeb79c7e573de0e5,BSI-DSZ-CC-0845-2012,None,None,None +af373587f2c57a24,BSI-DSZ-CC-0262-2005,None,None,None +af373587f2c57a24,CCEVS-VR-05-0131,None,None,None +af4c9b6e93062fde,ANSSI-CC-2010/03,None,None,None +af4c9b6e93062fde,ANSSI-CC-2017/47,None,None,None +af4c9b6e93062fde,ANSSI-CC-2017/49,None,None,None +af4c9b6e93062fde,BSI-DSZ-CC-0955-V2-2016,None,None,None +af4c9b6e93062fde,BSI-DSZ-CC-0973-V2-2016,None,None,None +af58ba642f4fc3d2,BSI-DSZ-CC-0891-V3-2018,None,None,None +af58ba642f4fc3d2,BSI-DSZ-CC-0980-2017,None,None,None +af78198822da03a7,ANSSI-CC-2010/03,None,None,None +afcf1165734847de,BSI-DSZ-CC-0207-2003,None,None,None +afe441f40a42865e,BSI-DSZ-CC-0902-2017,None,None,None +b0fd694500bfffab,BSI-DSZ-CC-0234-2004,None,None,None +b12b4e4eacad1497,BSI-DSZ-CC-0399-2007,None,None,None +b12b4e4eacad1497,BSI-DSZ-CC-0445-2007,None,None,None +b1401488f998c1f0,BSI-DSZ-CC-1136-2021,None,None,None +b15c099c68671895,ANSSI-CC-2017/49,None,None,None +b168db2651cbc1d6,ANSSI-CC-2010/02,None,None,None +b168db2651cbc1d6,ANSSI-CC-2012/69,None,None,None +b1b04d862f4c32af,BSI-DSZ-CC-0349-2006,None,None,None +b1b385d5bad724ed,BSI-DSZ-CC-1081-2020,None,None,None +b21be5dd6fdec83b,BSI-DSZ-CC-0965-2015,None,None,None +b2436dcf1c8e5dc5,BSI-DSZ-CC-0482-2008,None,None,None +b2436dcf1c8e5dc5,BSI-DSZ-CC-0528-2008,None,None,None +b24a14935edd51ad,BSI-DSZ-CC-0645-2010,None,None,None +b24a14935edd51ad,BSI-DSZ-CC-0750-2011,None,None,None +b252ff42ce0b7443,BSI-DSZ-CC-1040-2019,None,None,None +b2ba4d07e03b47ea,BSI-DSZ-CC-1178-V2-2022,None,None,None +b3855f3a6c06c01c,BSI-DSZ-CC-0645-2010,None,None,None +b4768c31e98b2170,BSI-DSZ-CC-0782-2012,None,None,None +b47b457f73c42c60,BSI-DSZ-CC-0918-2020,None,None,None +b47b457f73c42c60,BSI-DSZ-CC-0957-V2-2016,None,None,None +b49efd086851f84d,ANSSI-CC-2013/29,None,None,None +b49efd086851f84d,BSI-DSZ-CC-0555-2009,None,None,None +b49efd086851f84d,BSI-DSZ-CC-0645-2010,None,None,None +b4fc6a7b6c200b1b,BSI-DSZ-CC-1109-2019,None,None,None +b584e424a8b3dabe,BSI-DSZ-CC-0907-2013,None,None,None +b5968cd571a14cf3,BSI-DSZ-CC-0760-2013,None,None,None +b5968cd571a14cf3,BSI-DSZ-CC-0813-2012,None,None,None +b5985efd378af972,BSI-DSZ-CC-0897-V2-2014,None,None,None +b5b8bd0cb8bb7658,ANSSI-CC-2010/02,None,None,None +b5b8de155e4e40e6,ANSSI-CC-2015/36,None,None,None +b5b8de155e4e40e6,ANSSI-CC-2015/80,None,None,None +b644b78aafc367b0,BSI-DSZ-CC-0687-2011,None,None,None +b6a96b0dcf6eadf0,BSI-DSZ-CC-0955-V2-2016,None,None,None +b6c15035f759ec50,BSI-DSZ-CC-0269-2006,None,None,None +b6c3a9d53b9e784a,BSI-DSZ-CC-0750-V2-2014,None,None,None +b6c3a9d53b9e784a,NSCIB-CC-13-37760-CR2,None,None,None +b6e60b9025d116e4,ANSSI-CC-2014/46,None,None,None +b7132fe23fbdf9e1,BSI-DSZ-CC-0434-2007,None,None,None +b7132fe23fbdf9e1,BSI-DSZ-CC-0489-2008,None,None,None +b7209eec8fe7fc58,ANSSI-CC-2009/39,None,None,None +b764f5d5391e431e,BSI-DSZ-CC-0479-2008,None,None,None +b764f5d5391e431e,BSI-DSZ-CC-0482-2008,None,None,None +b804c318fc0c1e9b,BSI-DSZ-CC-0465-2008,None,None,None +b804c318fc0c1e9b,BSI-DSZ-CC-0550-2008,None,None,None +b82b48b0e915aaac,BSI-DSZ-CC-0169-2002,None,None,None +b883e389d34b1bb1,BSI-DSZ-CC-0410-2007,None,None,None +b883e389d34b1bb1,BSI-DSZ-CC-0417-2008,None,None,None +b883e389d34b1bb1,BSI-DSZ-CC-0521-2008,None,None,None +b887b7dcd6f1fdbb,BSI-DSZ-CC-0234-2004,None,None,None +b95f2b009d70fad1,BSI-DSZ-CC-0534-2009,None,None,None +b965c3b6a03aefc4,BSI-DSZ-CC-0348-2006,None,None,None +b99c572f58cfb19e,ANSSI-CC-2010/02,None,None,None +b99c572f58cfb19e,ANSSI-CC-2010/17,None,None,None +b99c572f58cfb19e,ANSSI-CC-2010/33,None,None,None +b9a6922899b66b8b,BSI-DSZ-CC-0813-2012,None,None,None +ba521df2c728c5a2,ANSSI-CC-2010/02,None,None,None +baead73ecc6e4a33,BSI-DSZ-CC-0421-2008,None,None,None +bb02df07a0d95e77,BSI-DSZ-CC-0322-2005,None,None,None +bb21cecb8bbfdf63,BSI-DSZ-CC-0269-2006,None,None,None +bb21cecb8bbfdf63,BSI-DSZ-CC-0376-2006,None,None,None +bb509db67c18bc40,BSI-DSZ-CC-0322-2005,None,None,None +bbc9a3a1d685d5f1,ANSSI-CC-2014/75,None,None,None +bc1452fff95744b9,BSI-DSZ-CC-0349-2006,None,None,None +bc25a76284e7c7ce,BSI-DSZ-CC-0782-2012,None,None,None +bc715a45d7584d95,BSI-DSZ-CC-0891-V2-2016,None,None,None +bd346dd3e46dff68,BSI-DSZ-CC-0639-2010,None,None,None +bdf8d5624e230986,BSI-DSZ-CC-0808-V2-2016,None,None,None +bdf8d5624e230986,BSI-DSZ-CC-0809-2013,None,None,None +bdf8d5624e230986,BSI-DSZ-CC-0978-2016,None,None,None +be9ca77bec616fe5,ANSSI-CC-2010/02,None,None,None +be9ca77bec616fe5,ANSSI-CC-2013/54,None,None,None +bec58e970e8cd6c8,BSI-DSZ-CC-0955-V2-2016,None,None,None +bf0f48ffd0dff903,BSI-DSZ-CC-0410-2007,None,None,None +bf0f48ffd0dff903,BSI-DSZ-CC-0463-2008,None,None,None +bf1338e9abe85c39,BSI-DSZ-CC-0410-2007,None,None,None +bff1330c13ccd99b,BSI-DSZ-CC-0293-2005,None,None,None +bff1330c13ccd99b,BSI-DSZ-CC-0346-2006,None,None,None +c06c7eb3afa4aea0,ANSSI-CC-2016/08,None,None,None +c10a3a62bf9fe782,BSI-DSZ-CC-1130-2021,None,None,None +c16f3dafab92df61,CCEVS-VR-05-0094,None,None,None +c1754dc2e7d666d8,BSI-DSZ-CC-0837-V2-2014,None,None,None +c18dc9f29967fe7f,BSI-DSZ-CC-0845-V2-2013,None,None,None +c1e14f36f031a342,BSI-DSZ-CC-1107-2020,None,None,None +c23218b044706800,BSI-DSZ-CC-0404-2007,None,None,None +c23218b044706800,BSI-DSZ-CC-0410-2007,None,None,None +c23218b044706800,BSI-DSZ-CC-0411-2007,None,None,None +c28289a45fc012a6,CRP210,None,None,None +c3dc8cfd97735115,BSI-DSZ-CC-0231-2004,None,None,None +c459c68e95d54f87,ANSSI-CC-2019/47,None,None,None +c459c68e95d54f87,ANSSI-CC-2020/26,None,None,None +c459c68e95d54f87,BSI-DSZ-CC-1059-2018,None,None,None +c459c68e95d54f87,BSI-DSZ-CC-1059-V3-2019,None,None,None +c483baec90d76587,BSI-DSZ-CC-0917-2014,None,None,None +c497da3fdb2f5027,ANSSI-CC-2017/49,None,None,None +c497da3fdb2f5027,BSI-DSZ-CC-1059-2018,None,None,None +c4ca6565698f0e49,BSI-DSZ-CC-1055-2018,None,None,None +c4e6a655a15144dd,BSI-DSZ-CC-0234-2004,None,None,None +c4f5a7748428e7d1,BSI-DSZ-CC-0349-2006,None,None,None +c518dd59d12daf17,BSI-DSZ-CC-0852-2013,None,None,None +c572cacbeb840621,BSI-DSZ-CC-0978-2016,None,None,None +c5bc8b961a199646,ANSSI-CC-2014/70,None,None,None +c621d1371b314913,BSI-DSZ-CC-0978-2016,None,None,None +c67724216ed5a318,BSI-DSZ-CC-0928-2018,None,None,None +c6c291cd525225b7,BSI-DSZ-CC-1025-2018,None,None,None +c8e381408b191a02,BSI-DSZ-CC-0410-2007,None,None,None +c8e381408b191a02,BSI-DSZ-CC-0417-2008,None,None,None +c8e381408b191a02,BSI-DSZ-CC-0575-2009,None,None,None +c8fc487eec95c21e,ANSSI-CC-2010/02,None,None,None +c902a788d699fe4a,SERTIT-003,None,None,None +c902a788d699fe4a,SERTIT-006,None,None,None +c96ac4c4015414ad,BSI-DSZ-CC-0782-V2-2015,None,None,None +ca1fa7c3f26bb3c5,BSI-DSZ-CC-0178-2003,None,None,None +ca63425a882729f3,BSI-DSZ-CC-0891-V4-2019,None,None,None +ca63425a882729f3,BSI-DSZ-CC-1033-V2-2021,None,None,None +ca63425a882729f3,BSI-DSZ-CC-1105-2020,None,None,None +ca88a34b0fcc5b0e,BSI-DSZ-CC-0280-2004,None,None,None +cb9a110799bc37f8,BSI-DSZ-CC-0954-2015,None,None,None +cbe5be4b63677c25,BSI-DSZ-CC-0752-2013,None,None,None +cc5a3d2a52f99ab3,ANSSI-CC-2018/25,None,None,None +cc5e2f58b681e555,BSI-DSZ-CC-0437-2008,None,None,None +cc5e2f58b681e555,BSI-DSZ-CC-0668-2010,None,None,None +cc64f3b8666b17bf,BSI-DSZ-CC-1072-2018,None,None,None +cc6918e95b803734,BSI-DSZ-CC-0891-V4-2019,None,None,None +cc6918e95b803734,BSI-DSZ-CC-1112-2020,None,None,None +cc932dbc293df39e,BSI-DSZ-CC-0973-2016,None,None,None +cc99b5d3806b1e97,ANSSI-CC-2016/66,None,None,None +cc99b5d3806b1e97,ANSSI-CC-2016/67,None,None,None +cc99b5d3806b1e97,BSI-DSZ-CC-0782-V2-2015,None,None,None +cc99b5d3806b1e97,BSI-DSZ-CC-0879-V2-2015,None,None,None +cce1a95634c6972b,BSI-DSZ-CC-0583-2009,None,None,None +cce214002aca738f,BSI-DSZ-CC-0897-V2-2014,None,None,None +cce214002aca738f,NSCIB-CC-16-99111,None,None,None +cd72fc2b62780a43,ANSSI-CC-2010/02,None,None,None +cd72fc2b62780a43,ANSSI-CC-2010/21,None,None,None +cdbbbad526f0aa13,BSI-DSZ-CC-1110-V4-2021,None,None,None +cdecc0eedcf05d2c,BSI-DSZ-CC-0285-2005,None,None,None +ce22219c2049985c,ANSSI-CC-2020/32,None,None,None +ce92195cf4a19084,BSI-DSZ-CC-0494-2008,None,None,None +cecb6a5d441fba98,BSI-DSZ-CC-0801-2012,None,None,None +cf1c544add4bf860,BSI-DSZ-CC-1137-2020,None,None,None +d063a93beba2dd65,BSI-DSZ-CC-0555-2009,None,None,None +d07b6e3036abb8f1,BSI-DSZ-CC-0645-2010,None,None,None +d07b6e3036abb8f1,BSI-DSZ-CC-0731-2011,None,None,None +d07b6e3036abb8f1,BSI-DSZ-CC-0741-2011,None,None,None +d07b6e3036abb8f1,BSI-DSZ-CC-0750-2011,None,None,None +d08b5ddedd34448d,ANSSI-CC-2018/35,None,None,None +d08b5ddedd34448d,BSI-DSZ-CC-0945-2017,None,None,None +d12a50d33b1c4253,CCEVS-VR-05-0094,None,None,None +d1fe94ee32cbbf05,BSI-DSZ-CC-0266-2005,None,None,None +d2bece2b0eebce5b,BSI-DSZ-CC-0879-V4-2020,None,None,None +d2bece2b0eebce5b,BSI-DSZ-CC-1071-V2-2019,None,None,None +d2d43918e5447c09,BSI-DSZ-CC-1156-V2-2021,None,None,None +d317c9c2e53ce427,BSI-DSZ-CC-0808-2013,None,None,None +d317c9c2e53ce427,BSI-DSZ-CC-0809-V2-2016,None,None,None +d317c9c2e53ce427,BSI-DSZ-CC-0978-2016,None,None,None +d32e5d4d5ff8b871,ANSSI-CC-2017/76,None,None,None +d32e5d4d5ff8b871,BSI-DSZ-CC-0891-V2-2016,None,None,None +d3323745a51a604d,BSI-DSZ-CC-0695-2011,None,None,None +d3323745a51a604d,BSI-DSZ-CC-0813-2012,None,None,None +d3323745a51a604d,BSI-DSZ-CC-0829-2012,None,None,None +d3420c2bf8974a5d,BSI-DSZ-CC-0782-V2-2015,None,None,None +d3420c2bf8974a5d,BSI-DSZ-CC-0967-2016,None,None,None +d34363168b1590bd,BSI-DSZ-CC-0964-V3-2017,None,None,None +d3987535a95fbffd,BSI-DSZ-CC-1040-2019,None,None,None +d3c2a59a2cffa30c,BSI-DSZ-CC-0555-2009,None,None,None +d3d257648d4c3105,ANSSI-CC-2010/02,None,None,None +d3d257648d4c3105,ANSSI-CC-2010/19,None,None,None +d3d257648d4c3105,ANSSI-CC-2010/56,None,None,None +d3d257648d4c3105,ANSSI-CC-2010/57,None,None,None +d4647f5217d84ba3,BSI-DSZ-CC-1040-2019,None,None,None +d4a1feebc1e1cb5d,BSI-DSZ-CC-0858-2013,None,None,None +d5187fc1cd590073,ANSSI-CC-2013/18,None,None,None +d67edade78d25033,ANSSI-CC-2009/27,None,None,None +d67edade78d25033,ANSSI-CC-2010/03,None,None,None +d69309b0a27c929c,BSI-DSZ-CC-1133-2020,None,None,None +d6ade0a63d0f9130,BSI-DSZ-CC-0897-V2-2014,None,None,None +d6ade0a63d0f9130,NSCIB-CC-12-36243-CR2,None,None,None +d6d129c94c368155,BSI-DSZ-CC-0217-2003,None,None,None +d71d0b082233758e,CRP260,None,None,None +d7501476836cf315,ANSSI-CC-2019/61,None,None,None +d75ff490c057793c,BSI-DSZ-CC-0727-2011,None,None,None +d80c26c2484dc87d,ANSSI-CC-2010/03,None,None,None +d9bfffa3cc6d1c53,BSI-DSZ-CC-0879-2014,None,None,None +da1139ecdc3512da,ANSSI-CC-2015/36,None,None,None +da2d3567a814d2d7,ANSSI-CC-2017/62,None,None,None +da2d3567a814d2d7,BSI-DSZ-CC-1059-V3-2019,None,None,None +da6845ee99cfa275,BSI-DSZ-CC-0266-2005,None,None,None +da6845ee99cfa275,BSI-DSZ-CC-0322-2005,None,None,None +da6845ee99cfa275,BSI-DSZ-CC-0399-2007,None,None,None +db28c1d7f201af1a,BSI-DSZ-CC-0640-2010,None,None,None +db28c203846775d8,ANSSI-CC-2014/06,None,None,None +db28c203846775d8,ANSSI-CC-2014/07,None,None,None +db28c203846775d8,BSI-DSZ-CC-0829-2012,None,None,None +db9267bdc717c801,BSI-DSZ-CC-0399-2007,None,None,None +db9267bdc717c801,BSI-DSZ-CC-0446-2007,None,None,None +dbbb56b48acbaf22,ANSSI-CC-2018/51,None,None,None +dbcf1311b3d95cdc,BSI-DSZ-CC-0332-2007,None,None,None +dbd62ef3bd952bad,BSI-DSZ-CC-0399-2007,None,None,None +dbe20d7c305b24eb,BSI-DSZ-CC-0750-V2-2014,None,None,None +dbe20d7c305b24eb,BSI-DSZ-CC-0858-2013,None,None,None +dbe20d7c305b24eb,NSCIB-CC-13-37762,None,None,None +dc51cf4e51f01086,NSCIB-CC-0351632-CR,None,None,None +dcd67613012044b3,BSI-DSZ-CC-0964-2015,None,None,None +dd4cfd03d9b3f25f,ANSSI-CC-2011/10,None,None,None +dd4cfd03d9b3f25f,BSI-DSZ-CC-0523-2008,None,None,None +dd4cfd03d9b3f25f,BSI-DSZ-CC-0626-2009,None,None,None +ddd9691cb063b07d,BSI-DSZ-CC-0973-V2-2016,None,None,None +ddd9691cb063b07d,BSI-DSZ-CC-0995-2018,None,None,None +ddd9691cb063b07d,BSI-DSZ-CC-0996-2018,None,None,None +de2ce0c1b72d5c90,BSI-DSZ-CC-0437-2008,None,None,None +de50abaf95e79d34,BSI-DSZ-CC-0813-2012,None,None,None +e0283686d26420d9,BSI-DSZ-CC-1021-2016,None,None,None +e0999b85d2f77733,BSI-DSZ-CC-0782-V2-2015,None,None,None +e0be32a08b66d264,BSI-DSZ-CC-0754-2012,None,None,None +e23f9c02819688f6,ANSSI-CC-2012/22,None,None,None +e247b50a301ce26c,BSI-DSZ-CC-1136-2021,None,None,None +e2a45ce49bb2b731,BSI-DSZ-CC-0829-2012,None,None,None +e2a45ce49bb2b731,BSI-DSZ-CC-0829-V2-2015,None,None,None +e3129aad84d5c40c,BSI-DSZ-CC-0837-V2-2014,None,None,None +e3231fac88242d81,BSI-DSZ-CC-0754-2012,None,None,None +e359ab2e436adf36,BSI-DSZ-CC-1040-2019,None,None,None +e359ab2e436adf36,BSI-DSZ-CC-1120-2020,None,None,None +e367ecaf05dacedb,BSI-DSZ-CC-0322-2005,None,None,None +e367ecaf05dacedb,BSI-DSZ-CC-0482-2008,None,None,None +e3a743ef04ecfb64,ANSSI-CC-2010/02,None,None,None +e4dc90de190437b8,CRP178,None,None,None +e4dc90de190437b8,CRP182,None,None,None +e501cafa3b025fb9,BSI-DSZ-CC-0750-2011,None,None,None +e501cafa3b025fb9,BSI-DSZ-CC-0858-2013,None,None,None +e501cafa3b025fb9,BSI-DSZ-CC-0866-2013,None,None,None +e501cafa3b025fb9,BSI-DSZ-CC-0866-V2-2014,None,None,None +e56d5d27cb2b3e6f,BSI-DSZ-CC-0403-2008,None,None,None +e56e14e8cbf89e3c,ANSSI-CC-2018/60,None,None,None +e56e14e8cbf89e3c,ANSSI-CC-2019/23,None,None,None +e5863682239df24b,BSI-DSZ-CC-0857-2013,None,None,None +e5b04a0a3872954f,ANSSI-CC-2011/07,None,None,None +e5b04a0a3872954f,ANSSI-CC-2012/48,None,None,None +e64266a9c13fc74e,ANSSI-CC-2014/81,None,None,None +e64266a9c13fc74e,BSI-DSZ-CC-0946-2014,None,None,None +e64266a9c13fc74e,BSI-DSZ-CC-0946-V2-2015,None,None,None +e64eedbb40d09a50,ANSSI-CC-2017/73,None,None,None +e6959c2b66202cb8,BSI-DSZ-CC-0891-V4-2019,None,None,None +e6aaffd588f2832e,ANSSI-CC-2013/56,None,None,None +e6aaffd588f2832e,ANSSI-CC-2014/83,None,None,None +e6dd3d435bd1b82e,BSI-DSZ-CC-0606-2010,None,None,None +e7a197d6aefb66df,BSI-DSZ-CC-0541-2010,None,None,None +e89a1a7817dae7e4,BSI-DSZ-CC-1089-2020,None,None,None +e8a37e48a8d34410,BSI-DSZ-CC-0466-2008,None,None,None +e8adf99d340489ce,ANSSI-CC-2017/76,None,None,None +e8adf99d340489ce,BSI-DSZ-CC-0891-V2-2016,None,None,None +e92ba97c2742e63d,ANSSI-CC-2019/33,None,None,None +e92ba97c2742e63d,ANSSI-CC-2020/26,None,None,None +e92ba97c2742e63d,BSI-DSZ-CC-1059-2018,None,None,None +e92ba97c2742e63d,BSI-DSZ-CC-1059-V3-2019,None,None,None +e93cb94a06c6957e,BSI-DSZ-CC-0964-V3-2017,None,None,None +e9845b4128afd538,BSI-DSZ-CC-1107-2020,None,None,None +e9c5d02aae54cdc5,BSI-DSZ-CC-0891-V4-2019,None,None,None +ea17788d9a9d12eb,CRP253,None,None,None +ea1b5ef156502d98,BSI-DSZ-CC-0862-2016,None,None,None +ea316d47c03f9fdf,BSI-DSZ-CC-0813-2012,None,None,None +ea330145d3ba738d,BSI-DSZ-CC-0322-2005,None,None,None +ea330145d3ba738d,BSI-DSZ-CC-0456-2009,None,None,None +ea4f0c0a7b82f526,BSI-DSZ-CC-0964-2015,None,None,None +eac2041efb4cf9f7,ANSSI-CC-2019/12,None,None,None +ec21d3c76c5ef65d,BSI-DSZ-CC-0904-V2-2021,None,None,None +ec21d3c76c5ef65d,BSI-DSZ-CC-1068-V2-2022,None,None,None +ec8a2a25cdbda634,ANSSI-CC-2020/07,None,None,None +ec8a2a25cdbda634,BSI-DSZ-CC-1110-V2-2019,None,None,None +ecee66b37d7576db,BSI-DSZ-CC-0244-2004,None,None,None +ed511caf8988dc45,BSI-DSZ-CC-0827-2013,None,None,None +ee35d872179a1397,BSI-DSZ-CC-0341-2006,None,None,None +ee35d872179a1397,BSI-DSZ-CC-0379-2006,None,None,None +ee3a996c18ace68e,BSI-DSZ-CC-0897-V2-2014,None,None,None +ee4e169753085913,BSI-DSZ-CC-0645-2010,None,None,None +ee4e169753085913,BSI-DSZ-CC-0682-2010,None,None,None +ee6718f7a7e3d30b,BSI-DSZ-CC-0266-2005,None,None,None +ee6718f7a7e3d30b,BSI-DSZ-CC-0338-2005,None,None,None +ef02bc67ebead2a2,BSI-DSZ-CC-0645-2010,None,None,None +ef02bc67ebead2a2,BSI-DSZ-CC-0675-2011,None,None,None +ef02bc67ebead2a2,BSI-DSZ-CC-0750-2011,None,None,None +f0bd6a29eee94a2c,ANSSI-CC-2018/08,None,None,None +f1300cb717ea9335,BSI-DSZ-CC-0900-2014,None,None,None +f14484e630df1dec,ANSSI-CC-2011/07,None,None,None +f14484e630df1dec,ANSSI-CC-2011/77,None,None,None +f14484e630df1dec,ANSSI-CC-2012/48,None,None,None +f14484e630df1dec,ANSSI-CC-2013/18,None,None,None +f1a73b2232787b26,ANSSI-CC-2010/02,None,None,None +f1a73b2232787b26,BSI-DSZ-CC-0664-2011,None,None,None +f1b9158cc2b98388,BSI-DSZ-CC-0782-2012,None,None,None +f1b9158cc2b98388,CRP279,None,None,None +f229a8078cb4ff82,BSI-DSZ-CC-1059-V2-2019,None,None,None +f2327302df1aa660,ANSSI-CC-2012/70,None,None,None +f2516946c797b1a0,ANSSI-CC-2013/42,None,None,None +f2516946c797b1a0,BSI-DSZ-CC-0829-2012,None,None,None +f2e7c6dfd6a431c8,BSI-DSZ-CC-0782-V5-2020,None,None,None +f2eb0ef953f3f147,ANSSI-CC-2015/45,None,None,None +f2eb0ef953f3f147,BSI-DSZ-CC-0782-V2-2015,None,None,None +f2fd56c195593dd8,ANSSI-CC-2020/94,None,None,None +f34df0dd7011366e,ANSSI-CC-2016/58,None,None,None +f34df0dd7011366e,ANSSI-CC-2017/61,None,None,None +f3bedaa5ffeeadb1,ANSSI-CC-2015/36,None,None,None +f4b4b026e87b5086,ANSSI-CC-2019/01,None,None,None +f4da9e13977580f4,BSI-DSZ-CC-0916-2015,None,None,None +f569fb2a6ddae452,BSI-DSZ-CC-0410-2007,None,None,None +f569fb2a6ddae452,BSI-DSZ-CC-0446-2007,None,None,None +f588fbe6fd36bc6b,BSI-DSZ-CC-0312-2005,None,None,None +f588fbe6fd36bc6b,BSI-DSZ-CC-0338-2005,None,None,None +f5a53fa6d2c3d9b2,ANSSI-CC-2017/11,None,None,None +f621ca835cf98a6d,BSI-DSZ-CC-0622-2012,None,None,None +f67f3737a3b47208,BSI-DSZ-CC-0782-2012,None,None,None +f68006f6c05e1211,BSI-DSZ-CC-0410-2007,None,None,None +f68006f6c05e1211,BSI-DSZ-CC-0411-2007,None,None,None +f731aaaab07bdca2,BSI-DSZ-CC-0973-2016,None,None,None +f731aaaab07bdca2,BSI-DSZ-CC-0973-V2-2016,None,None,None +f7941427d5fa40ea,BSI-DSZ-CC-0374-2006,None,None,None +f7cbb33cc639b5b5,BSI-DSZ-CC-0438-2007,None,None,None +f7cbb33cc639b5b5,BSI-DSZ-CC-0501-2008,None,None,None +f8a3fde3a557f44d,ANSSI-CC-2015/73,None,None,None +f8a3fde3a557f44d,BSI-DSZ-CC-0917-2014,None,None,None +f96bfb09bb1bd640,ANSSI-CC-2019/41,None,None,None +f96bfb09bb1bd640,ANSSI-CC-2020/07,None,None,None +f96bfb09bb1bd640,BSI-DSZ-CC-1110-V4-2021,None,None,None +f9d96d0eb2bc85cc,BSI-DSZ-CC-0750-V2-2014,None,None,None +f9d96d0eb2bc85cc,BSI-DSZ-CC-0858-2013,None,None,None +f9d96d0eb2bc85cc,BSI-DSZ-CC-0912-2014,None,None,None +f9d96d0eb2bc85cc,BSI-DSZ-CC-0913-2014,None,None,None +f9d96d0eb2bc85cc,BSI-DSZ-CC-0914-2014,None,None,None +f9d96d0eb2bc85cc,NSCIB-CC-13-37760-CR2,None,None,None +f9d9fe0dd9f7f110,BSI-DSZ-CC-0523-2008,None,None,None +f9e9366287e79e4d,BSI-DSZ-CC-0470-2008,None,None,None +f9fb8170cecedc23,BSI-DSZ-CC-1107-V3-2022,None,None,None +f9fb8170cecedc23,NSCIB-CC-0289060-CR,None,None,None +fa8384342c25ec59,BSI-DSZ-CC-0955-V2-2016,None,None,None +fb1fff87c87b82d6,ANSSI-CC-2012/70,None,None,None +fb1fff87c87b82d6,BSI-DSZ-CC-0894-2014,None,None,None +fb2cfdfef4f93536,BSI-DSZ-CC-0951-V2-2017,None,None,None +fb8010928a4bb56b,BSI-DSZ-CC-0837-V2-2014,None,None,None +fb8010928a4bb56b,BSI-DSZ-CC-0892-2014,None,None,None +fb8010928a4bb56b,BSI-DSZ-CC-0893-2014,None,None,None +fb8010928a4bb56b,NSCIB-CC-12-36243-CR2,None,None,None +fbdaf22c4f50391c,BSI-DSZ-CC-0891-V3-2018,None,None,None +fbe41655a05a01ff,BSI-DSZ-CC-0718-2012,None,None,None +fca98ecd003e1b82,ANSSI-CC-2009/11,None,None,None +fca98ecd003e1b82,BSI-DSZ-CC-0421-2008,None,None,None +fce7e01f2cd0cac0,ANSSI-CC-2011/10,None,None,None +fce7e01f2cd0cac0,ANSSI-CC-2011/12,None,None,None +fce7e01f2cd0cac0,BSI-DSZ-CC-0523-2008,None,None,None +fce7e01f2cd0cac0,BSI-DSZ-CC-0630-2010,None,None,None +fd3f988b7a505c45,BSI-DSZ-CC-0169-2002,None,None,None +fd583001a87023fd,BSI-DSZ-CC-0481-2008,None,None,None +fd583001a87023fd,BSI-DSZ-CC-0579-2009,None,None,None +fd583001a87023fd,CCEVS-VR-07-0054,None,None,None +fd966f807d3ba9fe,BSI-DSZ-CC-0813-2012,None,None,None +fd966f807d3ba9fe,BSI-DSZ-CC-0850-2013,None,None,None +ff0ffb9a32873433,BSI-DSZ-CC-0266-2005,None,None,None +ff0ffb9a32873433,BSI-DSZ-CC-0322-2005,None,None,None +ff45605151ec4eb1,ANSSI-CC-2013/39,None,None,None +ff45605151ec4eb1,BSI-DSZ-CC-0606-2010,None,None,None +ff45605151ec4eb1,BSI-DSZ-CC-0640-2010,None,None,None +ff45605151ec4eb1,BSI-DSZ-CC-0813-2012,None,None,None +ffc97a8fff838fa8,BSI-DSZ-CC-0266-2005,None,None,None +ffc97a8fff838fa8,BSI-DSZ-CC-0322-2005,None,None,None +ffc97a8fff838fa8,BSI-DSZ-CC-0538-2008,None,None,None +ffd213e55eee8cc8,BSI-DSZ-CC-0976-V2-2018,None,None,None +ffd213e55eee8cc8,BSI-DSZ-CC-1110-V2-2019,None,None,None +ffe4328dcdc6b538,BSI-DSZ-CC-0715-2011,None,None,None +fff289a66cb9ce4b,NSCIB-CC-0351632-CR,None,None,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv new file mode 100644 index 00000000..b29c609e --- /dev/null +++ b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv @@ -0,0 +1,101 @@ +dgst,referenced_cert_id,source,label,comment +e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,None,None,None +0bf7a19b22163465,ANSSI-CC-2016/44,None,None,None +ab3af998dff7a2ef,ANSSI-CC-2017/47,None,None,None +8ba22f6c9651edc3,ANSSI-CC-2013/55,None,None,None +680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,None,None,None +3515801dee00995f,BSI-DSZ-CC-0447-2008,None,None,None +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,None,None,None +12e20466a0f08342,ANSSI-CC-2016/79,None,None,None +d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,None,None,None +921e042759b30033,BSI-DSZ-CC-0978-2016,None,None,None +00efeb17bcaafce6,ANSSI-CC-2009/34,None,None,None +c175537338906951,ANSSI-CC-2016/70,None,None,None +fb9ec9e846ea4e87,ANSSI-CC-2017/47,None,None,None +ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,None,None,None +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,None,None,None +cacdfdafce47678c,ANSSI-CC-2014/59,None,None,None +dba4b5a166f9456a,ANSSI-CC-2014/76,None,None,None +aa096ffde94b953b,BSI-DSZ-CC-0411-2007,None,None,None +b42009eb34746731,ANSSI-CC-2021/50,None,None,None +463ecd64b7506048,ANSSI-CC-2018/52,None,None,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,None,None,None +c96343baf608174d,BSI-DSZ-CC-0645-2010,None,None,None +0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,None,None,None +2c15dfc106ebf8d8,ANSSI-CC-2012/24,None,None,None +e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,None +ee1c6dd97918d74a,ANSSI-CC-2012/71,None,None,None +e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,None,None,None +845bb039719ac5d8,ANSSI-CC-2010/40,None,None,None +af61a31e3fd0d6f0,ANSSI-CC-2010/02,None,None,None +d49988efd778ca9d,ANSSI-CC-2010/02,None,None,None +54273dd266fce692,ANSSI-CC-2015/15,None,None,None +4bede5d2af4cb11f,ANSSI-CC-2019/59,None,None,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,None,None,None +a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,None,None,None +f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,None,None,None +8bdb610131555c12,BSI-DSZ-CC-0645-2010,None,None,None +b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,None +371dca18821f7714,BSI-DSZ-CC-0945-2017,None,None,None +f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,None,None,None +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,None,None,None +4f5f41ecf7517e63,ANSSI-CC-2016/80,None,None,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,None,None,None +16bdbde359584f99,ANSSI-CC-2017/61,None,None,None +34370b67b5e675c3,ANSSI-CC-2013/47,None,None,None +a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,None,None,None +0bf7a19b22163465,ANSSI-CC-2015/80,None,None,None +1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,None,None,None +13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,None,None,None +f869956d14ea1694,ANSSI-CC-2012/68,None,None,None +037577fc2019fcfa,ANSSI-CC-2021/49,None,None,None +17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,None,None,None +5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,None,None,None +1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,None,None,None +28e70f46ba4394dd,ANSSI-CC-2012/08,None,None,None +61832cb4291c343f,BSI-DSZ-CC-0955-2016,None,None,None +42799014c183e1b3,BSI-DSZ-CC-0312-2005,None,None,None +627bfd69c2a831cf,ANSSI-CC-2014/37,None,None,None +aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,None,None,None +05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,None,None,None +9529eb793550093c,ANSSI-CC-2018/40,None,None,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,None,None,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,None,None,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,None,None,None +ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,None,None,None +30f71b100c5cebae,ANSSI-CC-2015/15,None,None,None +6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,None,None,None +06e505abb8dad1b8,ANSSI-CC-2017/54,None,None,None +317713948b4473b3,BSI-DSZ-CC-0798-2012,None,None,None +055985699aee1e09,ANSSI-CC-2010/02,None,None,None +b61363e51ed90c7a,ANSSI-CC-2019/12,None,None,None +691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,None,None,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,None,None,None +19c60923e31777e7,ANSSI-CC-2017/49,None,None,None +f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,None,None,None +c96343baf608174d,BSI-DSZ-CC-0750-2011,None,None,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,None,None,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,None,None,None +5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,None,None,None +4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,None,None,None +c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,None,None,None +ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,None,None,None +30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,None,None,None +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,None,None,None +ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,None,None,None +bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,None,None,None +97d00a9f198a6e57,ANSSI-CC-2015/66,None,None,None +744a7a202d909323,ANSSI-CC-2017/24,None,None,None +df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,None,None,None +6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,None,None,None +1b8705a0486b3be1,NSCIB-CC-0209053-CR,None,None,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,None,None,None +cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,None,None,None +27d11629261d8806,ANSSI-CC-2014/59,None,None,None +a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,None,None,None +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,None,None,None +c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,None +84a75b6fc33b669e,ANSSI-CC-2014/08,None,None,None +36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,None,None,None +01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,None,None,None +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,None,None,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv b/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv new file mode 100644 index 00000000..09368347 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv @@ -0,0 +1,101 @@ +dgst,referenced_cert_id,source,label,comment +a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,None,None,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,None,None,None +0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,None,None,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,None,None,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,None,None,None +774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,None,None,None +c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,None,None,None +a5adb726852f5cc5,ANSSI-CC-2009/37,None,None,None +4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,None,None,None +cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,None,None,None +24de96ea505af909,ANSSI-CC-2021/29,None,None,None +2793414918738c7f,BSI-DSZ-CC-0857-2013,None,None,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,None,None,None +d96438a7165e5d82,BSI-DSZ-CC-0410-2007,None,None,None +4b3f577896c80f8a,ANSSI-CC-2015/80,None,None,None +22388445fe620ac0,ANSSI-CC-2014/20,None,None,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,None,None,None +6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,None,None,None +120a49c0aa284f68,BSI-DSZ-CC-0813-2012,None,None,None +994ac7d986a99e50,BSI-DSZ-CC-0640-2010,None,None,None +16abf8ee0697e64f,ANSSI-CC-2014/61,None,None,None +f05de4db911ec8af,CRP243,None,None,None +38f79f1a48e7844f,ANSSI-CC-2017/47,None,None,None +fe55e99f4f3dc195,ANSSI-CC-2013/33,None,None,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,None,None,None +c23bb542201aa511,BSI-DSZ-CC-0829-2012,None,None,None +2532fe8c7214c485,BSI-DSZ-CC-0675-2011,None,None,None +2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,None,None,None +833ff87d69cff5db,383-4-184,None,None,None +c7c1215819127a1c,BSI-DSZ-CC-0322-2005,None,None,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,None,None,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,None,None,None +94fb86d31446e531,BSI-DSZ-CC-0958-2015,None,None,None +37bae9eaf342542b,BSI-DSZ-CC-0349-2006,None,None,None +5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,None,None,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,None,None,None +d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,None,None,None +0f53257eb74bee54,BSI-DSZ-CC-0417-2008,None,None,None +84d6a370e4f38ddc,ANSSI-CC-2019/12,None,None,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,None,None,None +d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,None,None,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,None,None,None +e6057ffc5085a192,ANSSI-CC-2014/48,None,None,None +db7627e56b8621b1,BSI-DSZ-CC-0639-2010,None,None,None +4b3f577896c80f8a,ANSSI-CC-2016/44,None,None,None +5f3c9c6bf76ba72d,ANSSI-CC-2018/51,None,None,None +22d6578b30f5a227,BSI-DSZ-CC-1107-2020,None,None,None +a0c38b4389ad7cf7,ANSSI-CC-2018/12,None,None,None +19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,None,None,None +f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,None,None +2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,None,None,None +3357c28c91cc2ed1,ANSSI-CC-2016/44,None,None,None +f6317f134f532696,ANSSI-CC-2017/24,None,None,None +234da54efbcf734b,BSI-DSZ-CC-0917-2014,None,None,None +1b20b1629ba7d3a6,ANSSI-CC-2012/72,None,None,None +2532fe8c7214c485,BSI-DSZ-CC-0633-2010,None,None,None +b1ef468658a9a0c1,NSCIB-CC-0448219-CR,None,None,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,None,None,None +edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,None,None,None +3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,None,None,None +0fe8df0e85116b61,ANSSI-CC-2010/33,None,None,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,None,None,None +b00bb452e7a8176c,ANSSI-CC-2016/57,None,None,None +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,None,None,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,None,None,None +63c1045055ec3b58,BSI-DSZ-CC-0266-2005,None,None,None +a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,None,None,None +4c017087976d05eb,ANSSI-CC-2020/45,None,None,None +f4d510a39278e687,ANSSI-CC-2018/32,None,None,None +c23bb542201aa511,ANSSI-CC-2014/14,None,None,None +625b243397001f01,BSI-DSZ-CC-1059-2018,None,None,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,None,None,None +000176284683faa2,BSI-DSZ-CC-0640-2010,None,None,None +66faa094a9333d1a,ANSSI-CC-2014/86,None,None,None +a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,None,None,None +031667f4e242da61,CCEVS-VR-07-0054,None,None,None +459539da9315ccac,ANSSI-CC-2021/30,None,None,None +69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,None,None,None +a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,None,None,None +50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,None,None,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,None,None,None +1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,None,None,None +0c7ef6c32cbdee47,ANSSI-CC-2017/61,None,None,None +8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,None,None,None +6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,None,None,None +4962c0d8106b8b5e,ANSSI-CC-2015/37,None,None,None +e6057ffc5085a192,BSI-DSZ-CC-0829-2012,None,None,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,None,None,None +720aab4f782f4c60,BSI-DSZ-CC-0555-2009,None,None,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,None,None,None +d55eb8c97d71e843,ANSSI-CC-2021/49,None,None,None +73e0b884cd04d9bc,NSCIB-CC-12-36243,None,None,None +2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,None,None,None +f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,None,None,None +f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,None,None,None +2f43fe24bd8e91e1,ANSSI-CC-2020/65,None,None,None +d96438a7165e5d82,BSI-DSZ-CC-0411-2007,None,None,None +7623950728f52a71,ANSSI-CC-2016/06,None,None,None +ba69597434946260,BSI-DSZ-CC-0293-2005,None,None,None +50aef770cb4f028f,BSI-DSZ-CC-1059-2018,None,None,None diff --git a/src/sec_certs/data/reference_annotations/outdated_manually_annotated_references.csv b/src/sec_certs/data/reference_annotations/outdated_manually_annotated_references.csv new file mode 100644 index 00000000..346ed3af --- /dev/null +++ b/src/sec_certs/data/reference_annotations/outdated_manually_annotated_references.csv @@ -0,0 +1,242 @@ +id,cert_id,location,reason,comment +61e7578e78fbe755,14.10.06/13-006,report,self, +dc91f94aa005a094,CCEVS-VR-VID10789-2017,report,self, +0349c893b81f7c1a,NSCIB-CC-0189590-CR,report,self, +0349c893b81f7c1a,NSCIB-CC-0031318-CR2,report,component used,"The TOE consists of an applet and the certified Java Card platform (SECORA ID-X Javacard OS platform) [PL-CERT]" +0349c893b81f7c1a,NSCIB-CC-0031318-CR2,target,component used,"intended to be used exclusively on the SECORA ID-X Javacard OS platforms which is certified according to CC EAL 5+ [Cert_Secora]" +0349c893b81f7c1a,BSI-DSZ-CC-1079-V2-2020,target,component used,"on a smart card chip based on the Infineon IFX_CCI_000010 security controller which is itself certified according to CC EAL 5+ [Cert_IC]." +ece463ca87392f7b,CCEVS-VR-06-0016,report,self, +c92a196d0408a5ff,ANSSI-CC-2013/57,report,self, +c92a196d0408a5ff,ANSSI-CC-2012/49,report,previous version,"previous version of the product (see [ANSSI-CC-2012/49])" +c92a196d0408a5ff,ANSSI-CC-2013/56,report,component used,"The main security services provided by the product are those provided by the previously certified (U)SIM platform see [ANSSI-CC2013/56]" +c92a196d0408a5ff,ANSSI-CC-2011/07,report,basis of eval,"The random number generator used by the final product was evaluated as part of the evaluation of the microcontroller (see [ANSSI-CC-2011/07])." +c92a196d0408a5ff,ANSSI-CC-2012/48,report,basis of eval,"TOE development phases taken into account during the evaluation of the platform (see [ANSSI-CC-2012/48])]" +0e22fe4e4e58faf4,BSI-DSZ-CC-1052-V5-2022,report,self, +0e22fe4e4e58faf4,BSI-DSZ-CC-1052-V4-2021,report,basis of recertification,"It is a re-certification based on BSI-DSZ-CC-1052-V4-2021" +0e22fe4e4e58faf4,BSI-DSZ-CC-1052-V5,target,self, +b4bec1e6f04c3533,BSI-DSZ-CC-0478-2008,report,self, +b4bec1e6f04c3533,BSI-DSZ-CC-0478,target,self, +940a18f62c79ffae,ANSSI-CC-2022/30,report,self, +940a18f62c79ffae,ANSSI-CC-2021/51,report,basis of eval,"The evaluation is based on the evaluation results of the product NPCT7xx TPM2.0 rev 1.59 (configuration version 1.3.0.0) certified under the reference [ANSSI-CC-2021/51]." +940a18f62c79ffae,ANSSI-CC-2022/30,target,self, +940a18f62c79ffae,ANSSI-CC-2021/51,target,basis of eval,"The evaluation is based on the evaluation results of the product NPCT7xx TPM2.0 rev 1.59 (configuration version 1.3.0.0) certified under the reference [ANSSI-CC-2021/51]." +00fc58baa6297c0d,C0072,report,self, +29964f32c68b0ce8,BSI-DSZ-CC-0519-V4-2022,report,self, +29964f32c68b0ce8,BSI-DSZ-CC-0519-V3-2021,report,basis of recertification,"The product ORGA 6141 online V3.8.2:1.2.0 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0519-V3-2021. Specific results from the evaluation process BSI-DSZ-CC-0519-V3-2021 were re-used." +29964f32c68b0ce8,BSI-DSZ-CC-0519-V4,target,self, +29964f32c68b0ce8,BSI-DSZ-CC-0519-V3,target,self, +b2f3874cdb97fe27,BSI-DSZ-CC-0790-2013,report,self, +1fb95703e4e12a49,383-4-263-CR,report,self, +546f9e2496b1d11c,ANSSI-CC-2015/14,report,self, +1dd7f4bf2e677073,ANSSI-CC-2014/20,report,self, +b24a14935edd51ad,BSI-DSZ-CC-0783-2013,report,self, +b24a14935edd51ad,BSI-DSZ-CC-0750-2010,report,component used,"The TOE consists of the Java Card Operating System the NXP Crypto Library for SmartMX v2.7 (certified under BSI-DSZ-CC-0750-2010) and the hardware platform P5CD145V0B or P5CC145V0B (certified under BSI-DSZ-CC-0645-2011)" +b24a14935edd51ad,BSI-DSZ-CC-0645-2011,report,component used,"The TOE consists of the Java Card Operating System the NXP Crypto Library for SmartMX v2.7 (certified under BSI-DSZ-CC-0750-2010) and the hardware platform P5CD145V0B or P5CC145V0B (certified under BSI-DSZ-CC-0645-2011)" +b24a14935edd51ad,BSI-DSZ-CC-0783,target,self, +b24a14935edd51ad,BSI-DSZ-CC-0750,target,component used,"Table 2 gives the details of the underlying evaluations of the cryptographic library and the underlying hardware platforms." +b24a14935edd51ad,BSI-DSZ-CC-0645,target,component used,"Table 2 gives the details of the underlying evaluations of the cryptographic library and the underlying hardware platforms." +302df01b5d463eea,CCEVS-VR-VID10253-2009,report,self, +6337d3dcc5362dc6,NSCIB-CC-0122197-CR,report,self, +6337d3dcc5362dc6,ANSSI-CC-2019/12,report,component used,Very unclear and does not actually list the target cert anywhere but in references. +6337d3dcc5362dc6,ANSSI-CC-2019/12,target,component used,"The ST31G480 Secured Microcontroller with Cryptographic Library has been certified by ANSSI (cert. report ANSSI-CC-2019/12) with assurance level EAL5+" +8da378edb071a71c,CCEVS-VR-VID10691-2016,report,self, +dd1482a4911a3f1d,CCEVS-VR-05-0105,report,self, +99223aca5d9eb3b3,ANSSI-CC-2011/01,report,self, +99223aca5d9eb3b3,BSI-DSZ-CC-0421-2008,report,component used,"This microcontroller was certified on December 4 2008 under the reference BSI-DSZ-CC-0421-2008." +99223aca5d9eb3b3,ANSSI-CC-2009-46,report,basis of eval,"The evaluation is based on the evaluation results of the ID-One Cosmo V7.0-a smart card product in Standard and Basic configuration certified on November 19 2009 under the reference ANSSI-CC-2009-46 [ANSSI-CC -2009-46]." +99223aca5d9eb3b3,BSI-DSZ-CC-0421-2008,target,component used,"IC Certificate" +99223aca5d9eb3b3,DCSSI-2009/11,target,component used,"Toolbox Certificate" +dc45c7a6ef17f8b5,BSI-DSZ-CC-1161-2020,report,self, +dc45c7a6ef17f8b5,BSI-DSZ-CC-1125-2019,report,basis of recertification,"This is a re-certification based on BSI-DSZ-CC-1125-2019. Specific results from the evaluation process BSI-DSZ-CC-1125-2019 were re-used." +dc45c7a6ef17f8b5,BSI-DSZ-CC-1161,target,self, +15fae84c1e6c6c57,ANSSI-CC-2001/05,report,self, +15fae84c1e6c6c57,ANSSI-CC-2001/05,report,component used,Very unclear and does not list the actual relationship directly. +eaf0033626287cc4,ANSSI-CC-2012/81,report,self, +3357e42cd4dc8605,383-4-256,report,self, +4e0dc3133a7e601d,C0513,report,self, +8877b69b21e92b41,2019-50-INF-3506,report,self, +3c05395ee9e5adf1,BSI-DSZ-CC-0707-2012,report,self, +a6e37f57c6e72d8c,ANSSI-CC-2005/13,report,self, +f0068ddc5569b772,518-LSS,report,self, +04715a4f818f4b71,383-4-345,report,self, +3c0185a5cb2d1ea8,CSEC2011001,report,self, +f91b60c2954d4b00,CSEC2017005,report,self, +b95f2b009d70fad1,BSI-DSZ-CC-0637-2010,report,self, +b95f2b009d70fad1,BSI-DSZ-CC-0534-2009,report,basis of recertification,"The product IBM z/OS Version 1 Release 11 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0534-2009. Specific results from the evaluation process BSI-DSZ-CC-0534-2009 were re-used." +b95f2b009d70fad1,BSI-DSZ-CC-0637,target,self, +d8c205b4924f91b3,ANSSI-CC-2022/09,report,self, +d8c205b4924f91b3,ANSSI-CC-2021/47,report,recertification,"The evaluation is based on the evaluation results of this same product certified on October 8, 2021 under the reference ANSSI-CC-2021/47, see [CER]. It corresponds to an evaluation with scope reduction following the identification of vulnerability." +82361ca3849ae025,383-4-420,report,self, +40817e88a5a46fd5,CCEVS-VR-VID10541-2014,report,self, +c1d88ce9dadd7d2d,BSI-DSZ-CC-0313-2006,report,self, +c1d88ce9dadd7d2d,BSI-DSZ-CC-0312-2005,report,component used,"The TOE is a Smart Device with an operating system (Starcos) and a dedicated file-system, that contains all data relevant for the ICAO application. For details on the MRTD chip and IC Dedicated Software see certification reports BSI-DSZ-CC-0312-2006 [13] for the Philips chip P5CT072V0N." +c1d88ce9dadd7d2d,BSI-DSZ-CC-0312-2005,target,component used,"HW-Part of TOE: Philips P5CT072V0N (BSI-DSZ-CC-0312-2005)." +ba27867a74e9c47a,CCEVS-VR-VID10472-2014,report,self, +f7cbb33cc639b5b5,KECS-ISIS-0222-2010,report,self, +f7cbb33cc639b5b5,BSI-DSZ-CC-0501-2008,report,component used,"The TOE KCOS e-Passport Version 1.1 S3CC9 LC/GC/GW is composed of the following components: IC chip S3CC9LC provided by Samsung Electronics see BSI-DSZ-CC-0501-2008" +f7cbb33cc639b5b5,BSI-DSZ-CC-0438-2007,report,component used,"The TOE KCOS e-Passport Version 1.1 S3CC9 LC/GC/GW is composed of the following components: IC chip S3CC9GC provided by Samsung Electronics see BSI-DSZ-CC-0438-2007" +f7cbb33cc639b5b5,BSI-DSZ-CC-0400-2007,report,component used,"The TOE KCOS e-Passport Version 1.1 S3CC9 LC/GC/GW is composed of the following components: IC chip S3CC9GW provided by Samsung Electronics see BSI-DSZ-CC-0400-2007" +f7cbb33cc639b5b5,BSI-DSZ-CC-0501-2008,target,component used,"IC Chip of Samsung Electronics and is certified according to CC EAL 5+(BSI-DSZ-CC-0501-2008)" +f7cbb33cc639b5b5,BSI-DSZ-CC-0438-2007,target,component used,"IC chip of Samsung Electronics and is certified according to CC EAL 4+(BSI-DSZ-CC-0438-2007)" +f7cbb33cc639b5b5,BSI-DSZ-CC-0400-2007,target,component used,"IC chip of Samsung Electronics and is certified according to CC EAL 4+(BSI-DSZ-CC-0400-2007)" +1e7b90184a127017,CCEVS-VR-VID10502-2012,report,self, +1e7b90184a127017,CCEVS-VR-VID10406-2011,report,previous version,"Sourcefire 3D System was Common Criteria certified by NIAP at EAL 2 augmented with ALC_FLR.2; Validation Report Number: CCEVS-VR-VID10406-2011" +0d77936727b14c88,BSI-DSZ-CC-0591-2011,report,self, +0d77936727b14c88,BSI-DSZ-CC-0591,target,self, +9060684220d85047,ANSSI-CC-2008/01,report,self, +9060684220d85047,BSI-DSZ-CC-0322,report,component used,"This evaluation thus took into account the results of the evaluation of the microcontroller (SLE66CX680PE - m1534-A13 with RSA2048 V1.4 & specific IC dedicated software) at the EAL5 level increased by the components ALC_DVS.2 AVA_MSU.3 and AVA_VLA.4 complies with the protection profile [PP0002]. This microcontroller was certified on September 14 2005 under the reference BSI-DSZ-CC-0322." +b23a595bb30865b5,C0575,report,self, +663d89dea15969c8,CRP214,report,self, +724d3996cb63d204,CSEC2014001,report,self, +724d3996cb63d204,CSEC2014001,target,self, +5e89afca1d4570a4,C0077,report,self, +37c8d23c44b95833,ANSSI-CC-2018/43,report,self, +37c8d23c44b95833,ANSSI-CC-2017/16,report,basis of eval,"Rating is based on product review results SAMSUNG S3FV9QM / S3FV9QK certified on March 23 2017 under the reference [ANSSI CC2017/16]." +dfb6af00c78cd061,C0421,report,self, +b9365d96c9ecdde0,CCEVS-VR-VID10869-2018,report,self, +c8427da3733c506d,OCSI/CERT/TEC/05/2017/RC,report,self, +780f19df05e0dd18,C0334,report,self, +56660124ad6ded6d,BSI-DSZ-CC-1089-2020,report,self, +56660124ad6ded6d,BSI-DSZ-CC-1089,target,self, +3ba9f330ce93636b,BSI-DSZ-CC-0732-2011,report,self, +3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,report,component used,"The TOE TCOS Signature Module Version 1.0 Release 1/SLE78CLX480P is composed of the components: Smart Card Security Controller SLE78CLX480P / M7820 A11 provided by Infineon Technologies AG (with specific IC dedicated software see BSI-DSZ-CC-0640-2010, but without usage of the Infineon crypto library)" +3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,report,basis of eval,"The evaluators have tested nearly the same TOE in the certification process BSI-DSZ-CC-0677 systematically against high attack potential during their penetration testing. They have analysed the changes between the two TOEs and found no differences in the source code concerning the security functionality of the TOE." +3ba9f330ce93636b,BSI-DSZ-CC-0732-2011,target,self, +3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,report,component used,"The evaluation of the TOE is a composite evaluation and uses the results of the CC evaluation provided by [HWCR]" +80932fd09bfc3f98,CCEVS-VR-VID10120-2008,report,self, +8298c7814b3b2860,KECS-CISS-1153-2022,report,self, +c290ee3692a00006,383-4-438,report,self, +642d82f9a5e97fa2,CSEC2017014,report,self, +642d82f9a5e97fa2,CSEC2017014,target,self, +4b2b04554256b523,KECS-ISIS-0117-2008,report,self, +4eaceb72085e12f7,C0668,report,self, +70b12d34fa092eeb,CCEVS-VR-VID10380-2012,report,self, +ec7db499df27dcf9,383-4-397,report,self, +61821500cddab34f,BSI-DSZ-CC-0540-2009,report,self, +61821500cddab34f,BSI-DSZ-CC-0540,target,self, +0f3900cdcd0c7f3e,NSCIB-CC-127667-CR3,report,self, +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,report,component used,"Certification Report Crypto Library V3.1x on P6021yVB NSCIB-CC-66030-CR5 7 July 2021" +0f3900cdcd0c7f3e,BSI-DSZ-CC-1072-V4-2021-MA-01,report,component used,"Certification Report NXP Secure Smart Card Controller P6021yVB including IC dedicated software BSI-DSZ-CC-1072-V4-2021-MA-01 17 December 2021" +0f3900cdcd0c7f3e,NSCIB-CC-127667,target,self, +0f3900cdcd0c7f3e,BSI-DSZ-CC-1072-V4-2021,target,component used,"The circuitry of the MRTD‘s chip P60D081PVB is certified under certificate identification BSI-DSZ-CC-1072-V4-2021" +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,target,component used,"and the Crypto Library V3.1.2 on P60D081PVB is certified under certificate identification NSCIB-CC-66030-CR5." +238f8edc5eda1358,BSI-DSZ-CC-0285-2005,report,self, +238f8edc5eda1358,BSI-DSZ-CC-0222-2003,report,basis of recertification,"The product IBM Tivoli Access Manager for e-Business, Version 5.1 with Fixpack 6 has undergone the certification procedure at BSI. This is a basis of recertification based on BSI-DSZ-CC-0222-2003." +b5bcac5eb0f61dfe,CCEVS-VR-10822-2018,report,self, +28fb117adbb809e3,CSEC2006002,report,self, +c10773ac43ca7c2b,CRP180,report,self, +6d6ade44dcc497dd,BSI-DSZ-CC-0312-2005,report,self, +6d6ade44dcc497dd,BSI-DSZ-CC-0227-2004,report,basis of recertification,"This is a re-certification based on BSI-DSZ-CC-0227-2004 and BSI-DSZ-CC-0227-2004-MA-01." +6d6ade44dcc497dd,BSI-DSZ-CC-0312,target,self, +c23ca69c2f7528d1,ANSSI-CC-2020/24,report,self, +abe89a00b3efa393,C0261,report,self, +cf17d0efe1278432,ANSSI-CC-2007/24,report,self, +7607e34a7608222c,ISCB-3-RPT-C098-CR-v1,report,self, +5ae867bded9ef672,C0180,report,self, +3f141b547bd4735f,CCEVS-VR-VID10571-2014,report,self, +ed0b1d4c39cbd251,C0727,report,self, +abc1f0f55adc2967,CCEVS-VR-VID11049,report,self, +d68511e69820f461,CRP260,report,self, +59b6bc42781e26d6,CSEC2019006,report,self, +d9d0986019774924,KECS-NISS-0050-2006,report,self, +318dcee9d8f6ac99,CSEC2018007,report,self, +318dcee9d8f6ac99,CSEC2018007,target,self, +a6fac58198296194,BSI-DSZ-CC-0857-2013,report,self, +a6fac58198296194,BSI-DSZ-CC-0555-2009,report,basis of recertification,"This is a re-certification based on BSI-DSZ-CC-0555-2009. Specific results from the evaluation process BSI-DSZ-CC-0555-2009 were re-used." +a6fac58198296194,BSI-DSZ-CC-0857,target,self, +f1a73b2232787b26,BSI-DSZ-CC-0671-2011,report,self, +f1a73b2232787b26,BSI-DSZ-CC-0664-2011,report,basis for,"The further security mechanisms Active Authentication and Extended Access Control as well as Chip Authentication are subjects of BSI-DSZ-CC-0664-2011" +f1a73b2232787b26,ANSSI-CC-2010/02,report,component used,"The TOE is a composite product. It is composed from an Integrated Circuit... For details concerning the CC evaluation of the STMicroelectronics IC see the evaluation documentation under the Certification ID ANSSI-CC-2010/02 [16]" +f1a73b2232787b26,BSI-DSZ-CC-0664-2011,target,basis for, +f1a73b2232787b26,ANSSI-CC-2010/02,target,component used,"SB23YR80B is certified according to CC EAL5 augmented (ANSSI-CC-2010/02)" +90f1d708d20deb30,SERTIT-043,report,self, +faecd563eb8f1d6c,C0314,report,self, +f5d49c63046c69d2,CCEVS-VR-VID10272-2008,report,self, +f2e00a1fee27bdd7,BSI-DSZ-CC-0957-2015,report,self, +f2e00a1fee27bdd7,BSI-DSZ-CC-0845-V2-2013,report,component used,"The TOE is set up as a composite product. It is composed from the Integrated Circuit... For details concerning the CC evaluation of the underlying IC see the evaluation documentation under the Certification ID BSI-DSZ-CC-0845-V2-2013" +f2e00a1fee27bdd7,BSI-DSZ-CC-0957,target,self, +33cb152e0a1be23c,CCEVS-VR-10928-2019,report,self, +c16a92b40550193d,383-4-170,report,self, +1fb1564dfb0f0b04,ANSSI-CC-2021/26,report,self, +1fb1564dfb0f0b04,ANSSI-CC-2020/34,report,component used,"This evaluation thus took into account the results of the evaluation of the PEGASUS_CB_05 microcontroller at the EAL6 level increased by the ASE_TSS.2 component conforming to the protection profile [PP0084]. This microcontroller was certified on September 18 2020 see [ANSSI-CC-2020/46]" +8b3c4eb3efd1cdaa,574-LSS,report,self, +14ddeb255e0a1cd2,2017-60-INF-2765,report,self, +01e3716dd2a5b510,BSI-DSZ-CC-0466-2008,report,self, +01e3716dd2a5b510,BSI-DSZ-CC-0466,target,self, +0c7ef6c32cbdee47,BSI-DSZ-CC-1073-2019,report,self, +0c7ef6c32cbdee47,BSI-DSZ-CC-1074-2019,report,basis for,"The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-1074-2019" +0c7ef6c32cbdee47,ANSSI-CC-2017/61,report,component used,"The TOE is a composite product. It is composed from an Integrated Circuit... For details concerning the CC evaluation of the STM IC and its cryptographic libraries see the evaluation documentation under the Certification ID ANSSI-CC-2017/61" +0c7ef6c32cbdee47,BSI-DSZ-CC-1073,target,self, +0c7ef6c32cbdee47,ANSSI-CC-2017/61,target,component used,"The IC hardware ST31G480 C01 including the cryptographic library Neslib 5.2.0 provides basic functionality, among others the contactless communication interface and low-level security features (e.g. random number generator 3DES- AES and EC-support). It is certified (ANSSI-CC-2017/61) according to CC EAL5" +13e3c6a0cd06ecf6,383-4-340,report,self, +814a2ac0aeae08b1,CCEVS-VR-03-0044,report,self, +11708925f70ce5d3,Certificate Number: 2011/72,report,self, +ca5da2fe138af656,BSI-DSZ-CC-0423-2007,report,self, +ca5da2fe138af656,BSI-DSZ-CC-0413-2007,report,basis of recertification,"The product S-TRUST Sign-it base components 2.1 Version 2.1.5.1 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0413-2007." +ca5da2fe138af656,BSI-DSZ-CC-0423,target,self, +98937f99289f2499,NSCIB-CC-15-66461-CR,report,self, +c3b443f243b95913,CCEVS-VR-11219-2021,report,self, +183a5cd3e2aaa0ec,ANSSI-CC-2017/81,report,self, +183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,report,component used,"The product consists of the following components: the previously certified M7892 G12 component (see [BSI-DSZ-CC-0891-V2-2016])" +183a5cd3e2aaa0ec,ANSSI-CC-2017/76,report,component used,"The product consists of the following components: an operating system in the form of an open configuration platform or closed Java Card MultiApp V4.0.1. This platform is certified under the reference [ANSSI-CC-2017/76]" +183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,target,component used,"The evaluation is a composite evaluation and uses the results of the CC evaluation provided by [CR-IC]" +18bb29b147e80caa,BSI-DSZ-CC-1040-2019,report,component used,"et du microcontrôleur « NXP Secure Smart Card Controller N7121 avec son firmware et sa bibliothèque cryptographique dédiés » (certifié sous la référence BSI-DSZ-CC- 1040, voir [CER_IC])" +d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,target,component used, +14df54305cad83b4,BSI-DSZ-CC-0385-2006,report,basis of recertification,"The product AIX 6 version 6100-00-02 with optional Virtual I/O Server (VIOS) version 1.5 has undergone the certification procedure at BSI. This is a re-certification based on BSI- DSZ-CC-0385-2006. Specific results from the evaluation process BSI-DSZ-CC-0385-2006 were re-used." +055985699aee1e09,ANSSI-CC-2010/02,report,component used, +a9639521d8a5ab09,ANSSI-CC-2010/02,report,component used,"The evaluation has been performed according to the composition scheme as defined in the guide [COMP] in order to assess that no weakness comes from the integration of the software in the microcontroller already certified. Therefore, the results of the evaluation of the microcontroller SB23YR80B with Neslib version 3.0” at EAL6 level augmented with ALC_FLR.1, compliant with the [PP0035] protection profile, have been used. This microcontroller has been certified the 10 February 2010 under the reference ANSSI-CC-2010/02. The maintenance report ANSSI-2010/02-M01 has also been issued the 19th March 2010 for this product." +5f1bc82727f54376,ANSSI-CC-2021/29,report,component used, +317713948b4473b3,BSI-DSZ-CC-0710-2010,report,NaN, +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,basis of eval,"Also, specific ALC related parts were re-used in the sense of AIS38 from BSI-DSZ-CC-0891-V2-2016 (Infineon Security Controller M7892 D11 and G12)." +b0b7b073ca2dfe5f,ANSSI-CC-2019/28,report,component used, +25eade3365f92578,BSI-DSZ-CC-0338-2005,report,component used, +b4a2999b439b1ba8,BSI-DSZ-CC-1178-2021,report,basis of recertification,"The product Infineon Technologies AG OPTIGATM Trusted Platform Module SLB9672_2.0 v16.10.16488.00 and SLB9673_2.0 v26.10.16688.00, has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-1178-2021. Specific results from the evaluation process BSI-DSZ-CC-1178-2021 were re-used." +850f5b2c312299d5,BSI-DSZ-CC-0330-2007,report,basis of recertification,"Das Produkt Virtuelle Poststelle des Bundes, (OSCI) Version 2.2.3.2 hat das Zertifizierungsverfahren beim BSI durchlaufen. Es handelt sich um eine Re-Zertifizierung basierend auf BSI-DSZ-CC-0330-2007." +e9e12b044b495c6f,ANSSI-CC-2010/01,report,component used, +aea11fd4d2a7709d,BSI-DSZ-CC-0808-V2-2016,report,basis of recertification,"TCOS Passport Version 2.1 Release 3/P60D144 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0808-V2-. Specific results from the evaluation process BSI-DSZ-CC-0808-V2-2016 were re-used" +845bb039719ac5d8,BSI-DSZ-CC-0645-2010,target,component used, +4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component used, +1b0a6ecee1f830fc,ANSSI-CC-2011/07,report,component used, +9ae8bef18938d2df,BSI-DSZ-CC-0965-2015,report,basis of recertification,"The product Infineon Technologies AG Trusted Platform Module SLB9670_2.0, v7.40.2098.00 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0965-2015. Specific results from the evaluation process BSI-DSZ-CC-0965-2015 were re-used." +7e82a1ceb29e4019,NSCIB-CC-12-36243,report,self, +54dd28994604ee35,BSI-DSZ-CC-0481-2008,NaN,NaN, +1357546ec01b1a65,BSI-DSZ-CC-0438-2007,report,basis of recertification,"The product S3CC91C 16-Bit RISC Microcontroller for Smart Card, Version 0 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0438-2007 . Specific results from the evaluation process based on BSI-DSZ-CC-0438-2007 were re-used." +30120e4f3aa2f30a,BSI-DSZ-CC-0897-2013,target,component used, +27b4366a0f1fcdef,CCEVS-VR-07-0054,report,basis for, +27b4366a0f1fcdef,BSI-DSZ-CC-0403-2008,report,basis of recertification,"The product Oracle Database 11g Standard Edition and Standard Edition One, Release 11.1.0.7 with Critical Patch Updates up to and including July 2009 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0403-2008. Specific results from the evaluation process BSI-DSZ-CC-0403-2008 were re-used." +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,basis of recertification,"This is a re-certification based on BSI-DSZ-CC-0961-V2-2018. Specific results from the evaluation process BSI-DSZ-CC-0961-V2-2018 were re-used. Also, specific ALC related parts were re-used in the sense of AIS38 from BSI-DSZ-CC-1079." +05a8a84b60a30ceb,BSI-DSZ-CC-1110-V3-2020,report,component used,"For details concerning the CC evaluation of the underlying IC see the evaluation documentation under the Certification ID BSI-DSZ-CC-1110-V3-2020 ([16], [17])." +1b797b27b981aafe,BSI-DSZ-CC-1059-2018,report,component used,"- du microcontrôleur « NXP P60D145 » certifié sous la référence [CER-IC] ;" +691119fe8a2f8e1c,BSI-DSZ-CC-0944-2014,report,basis of recertification,"The product SLS 32TLC00xS(M) CIPURSETM4move, v1.00.00 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0944-2014. Specific results from the evaluation process BSI-DSZ-CC-0944-2014 were re-used." +0d164fb0d72dfa00,BSI-DSZ-CC-0645-2010,report,component used, +a154a01ff4cf04c7,BSI-DSZ-CC-1059-2018,report,component used, +537154d2c4344d48,ANSSI-CC-2018/31,report,component used, +265e576fd3517f91,BSI-DSZ-CC-0866-2013,report,basis of recertification,"The product IDeal Pass v2 - SAC/EAC JC ePassport 4.0.0 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0866-2013. Specific results from the evaluation process BSI-DSZ-CC-0866-2013 were re-used." +38f79f1a48e7844f,ANSSI-CC-2017/49,report,component used, +66faa094a9333d1a,ANSSI-CC-2014/86,NaN,NaN, +b8875373ee66e841,ANSSI-CC-2011/07,report,component used, +3a3ae16e94bc7aba,BSI-DSZ-CC-0995-2018,report,basis of eval,"Please note that in consistency to the claimed protection profile BSI-CC-PP-0055-2009 the security mechanism Basic Access Control is in the focus of this evaluation process. The further security mechanisms Password Authenticated Connection Establishment, Extended Access Control and Active Authentication are subject of the separate evaluation process BSI-DSZ-CC-0995-2018 [25]." +c18fea56107696a8,BSI-DSZ-CC-0737-2012,report,basis of recertification,"The product Bundesdruckerei Document Reading Application, Version 1.2.1129 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0737-2012. Specific results from the evaluation process BSI-DSZ-CC-0737-2012 were re-used." +0694786160024e30,BSI-DSZ-CC-0990-2016,report,basis of recertification,"The product SLS 32TLC00xS(M) CIPURSETM 4move, V1.0.2 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0990-2016. Specific results from the evaluation process BSI-DSZ-CC-0990-2016 were re-used." +c63e878cc60a8682,BSI-DSZ-CC-0883-2013,target,component used,"The HW used to support the OS is the M7794 A12 and G12 (SLE77CLFX2407PM) component. This component M7794 A12 and G12 (BSI-DSZ-CC-0883-2013) is certified CC EAL5+ [9]." +50aef770cb4f028f,BSI-DSZ-CC-1059-2018,report,component used, +425f13fb7e94add5,ANSSI-CC-2020/21,report,component used, +d472c3dfb2f2a810,ANSSI-CC-2011/24,report,previous version,"Par ailleurs, cette évaluation a également pris en compte les résultats de l’évaluation de la version précédente du produit certifié (voir [ANSSI-CC-2011/24])." +d5074e4b714ac9e7,ANSSI-CC-2011/07,report,component used, +629372a8091eafae,ANSSI-CC-2020/71,report,component used, +cefb1202ed3cd997,BSI-DSZ-CC-0344-2005,NaN,NaN, +34de7910ace6d2e9,BSI-DSZ-CC-0410-2007,report,basis for, +0fe8df0e85116b61,ANSSI-CC-2010/35,report,previous version, +43109d44111df5d3,BSI-DSZ-CC-0915-2016,report,component used, +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,report,basis of recertification,"The product Infineon Technologies Smart Card IC (Security Controller) M5072 G11 with optional RSA v1.03.006, EC v1.03.006 and Toolbox v1.03.006 with specific IC dedicated software has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0946-2014. Specific results from the evaluation process BSI-DSZ-CC-0946-2014 were re-used." +a0c38b4389ad7cf7,ANSSI-CC-2018/12,report,component used, +5c06e4fb1887fdc7,BSI-DSZ-CC-0968-2016,report,basis of eval, +0bb56716a1fbef1e,ANSSI-CC-2019/35,report,component used, +83bc5fbf2b54e83b,BSI-DSZ-CC-0786-2012,report,basis of recertification,"The product Infineon smartcard IC (Security Controller) M7794 A12 with optional RSA2048/4096 v1.02.013, EC v1.02.013 and Toolbox v1.02.013, has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0786-2012. Specific results from the evaluation process BSI-DSZ-CC-0786-2012 were re-used." +2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,report,component used, +720aab4f782f4c60,BSI-DSZ-CC-0555-2009,report,component used, +0bae4647efd39e51,BSI-DSZ-CC-1190-2022,report,basis of recertification, +647d17d44745a532,BSI-DSZ-CC-0904-2015,report,basis of recertification,"The product TCOS FlexCert Version 2.0 Release 2/SLC52 has undergone the certification procedure at BSI. This is a re-certification based on BSI-DSZ-CC-0904-2015. Specific results from the evaluation process BSI-DSZ-CC-0904-2015 (including subsequent maintenance procedures BSI-DSZ-CC-0904-2015-MA-01 and BSI-DSZ-CC-0904-2015- MA-02) were re-used." +8927428c96c0b59a,BSI-DSZ-CC-0813-2012,report,component used, +8928e191ee2fd3b9,BSI-DSZ-CC-0626-2009,report,component used, +c23bb542201aa511,ANSSI-CC-2014/06,report,component used, diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md new file mode 100644 index 00000000..c17c55dd --- /dev/null +++ b/src/sec_certs/data/reference_annotations/readme.md @@ -0,0 +1,51 @@ +# Reference annotations + +This folder contains data and the methodology (presented below) related to learning the reference annotations. + +- The folder [split](split) contains split of the CC Dataset to `train/valid/test` splits for learning. +- The csv file [manually_annotated_references.csv](./manually_annotated_references.csv) contains manually acquired labels to references obtained with the methodology outlined below. + +### Reference taxonomy + +After manually inspecting random certificates, we have identified the following reference meanings: + +- **Component used**: The referenced certificate is a component used in the examined certificate (e.g., IC used by a smartcard). Some evaluation results were likely shared/re-used. This is a specific case of *evaluation reused* label +- **Component shared**: The referenced certificate shares some components with the examined certificate. Some evaluation results were likely shared/re-used. This is a specific case of *evaluation reused* label. +- **Evaluation reused**: The evaluation results of the referenced certificate were used for evaluation of the examined certificate, due to reasons that could not be resolved as *component used* nor *component shared*. +- **Recertification**: The examined certificate is a re-certification of the referenced certificate. +- **Previous version**: The product in the referenced certificate is a previous version of the product in the examined certificate and the recertification is not explicitly mentioned. +- **On platform**: The examined certificate runs on a platform that is certified in the referenced certificate. +- **Self**: The referenced certificate is the same as the examined certificate. + + +### Reference classification methodology + +**Data splits and manual annotations**: + +1. Inspect random certificates (>100) and capture the observed relations into reference taxonomy +2. Split all certificates for which we register a direct outgoing reference in either security target or certification report into `train/valid/test` splits in `30/20/50` fashion. + - See [split](split/) +3. Choose all samples from test set, random 100 samples from train set and random 100 samples from validation set for manual annotations. Store those into [manual_annotations](manual_annotations/) +4. Label all references in [manual_annotations](manual_annotations/) as follows: + - The reference meaning is recovered based on certification report and security target pdf. + - The annotator should visit certification report first. If data in certification report is ambigous or does not allow precise annotation, the annotator should further visit the security target pdf. + - The annotator should set `None` label on instances that he/she was unable to decide. + - The annotator should label the `source` as: + - `report` if he/she decided purely based on the data from certification report + - `target` if he/she decided purely based on the data from security target + - `report+target` if both artifacts were needed for resolution + - Note that this label is only internal, not intended to be used for research + - The label should be in lowercase letters with underscore, e.g., `evaluation_reused`. + - Artifacts of the referenced certificate are not examined. + - The labeling is done by a pair of co-authors, the inter-annotator agreement is measured with Cohen's Kappa + +**Learning the annotations**: + +1. For each pair `(dgst, referenced_cert_id)`, recover the relevant segments both from certification report and security target that mention the `referenced_cert_id` +2. Apply text processing on the segments (e.g., unify re-certification vs. recertification, etc.) +3. Train a baseline model based on TF-IDF (or count vectorization in general), random forest, and a soft-voting layer on top of that. + - Random forest classifies single segment to a probability of a given label + - Soft voting compares probabilities of the given labels on all segments, takes their square and chooses the maximum. +4. Train a sentence transformer with the same soft-voting layer on top of that. +5. Finetune hyperparameters. +6. Evaluate on test set. diff --git a/src/sec_certs/data/reference_annotations/split/test.json b/src/sec_certs/data/reference_annotations/split/test.json new file mode 100644 index 00000000..41ac06ee --- /dev/null +++ b/src/sec_certs/data/reference_annotations/split/test.json @@ -0,0 +1,780 @@ +[ + "11a98c5d5ff6f40f", + "6dde54b8c6d592ef", + "054bd8196e015197", + "c902a788d699fe4a", + "5d674c06a559a5e7", + "9f8c9a3aa64a9f8e", + "9545f2b0b8b92c8d", + "56ceb1a9c003ac3f", + "9299f6cd8a266541", + "d1fe94ee32cbbf05", + "6313a92bf426d2bb", + "e8a37e48a8d34410", + "f34df0dd7011366e", + "c23218b044706800", + "3f4b6e4f245f6fab", + "5f99b00bf258f32f", + "4f3bf0c433b7f54f", + "587055293e9fa51d", + "da2d3567a814d2d7", + "915e919390e6f58b", + "a5e8267322d0d7a6", + "82dce1546c69369d", + "7d4585a4b5b6e873", + "736ede0289146b59", + "0cbcee9f1cde47a8", + "4bf7d6ef245dd393", + "3817c0aca007989e", + "baead73ecc6e4a33", + "19d3185c40be008d", + "8a7731e9eb363fb4", + "78a279b07a6d3367", + "47913a485c3c8a18", + "5f259ce91f0234a0", + "d4a14ea7adb375ed", + "994a2bf994b4facd", + "ac1257069b2f4afd", + "b5985efd378af972", + "bc1452fff95744b9", + "4f953ae00a24eb85", + "a35ed74b7ea3ec4b", + "654554952ecb20ac", + "157b741a0f529a0a", + "30ae700d801bd41c", + "2f9feaf4121720da", + "e0283686d26420d9", + "1b2764a62ffe86a2", + "b47b457f73c42c60", + "ff45605151ec4eb1", + "5346d5a4418466ab", + "e5b04a0a3872954f", + "981f7ea91d7bdcb6", + "f229a8078cb4ff82", + "731aac44217611ec", + "4e29916c366fe997", + "5efe98a1ba4df4d7", + "ffd213e55eee8cc8", + "7e24168630d5124e", + "00ad121fa63e5dfb", + "173097995b7a7f12", + "4b2f963b48e0f954", + "5e4c757231135ba1", + "aed7a612d966a786", + "1468948b4f0509ab", + "4d885e54f1361d04", + "8a5f8686ab9de19e", + "37d28734245a3bb1", + "80fe3ba3d9c9bf64", + "b1b385d5bad724ed", + "c4ca6565698f0e49", + "d9bfffa3cc6d1c53", + "d32e5d4d5ff8b871", + "8865aed38c79d3fc", + "5bf118ffa29520c0", + "6136acacc653106c", + "05d96aada7bb839c", + "0aad0176e35b3749", + "64ed7e55bdc50177", + "3976c9e492193315", + "65fba51313f65938", + "55e52bcaab69d573", + "4dd0daec78093d31", + "d67edade78d25033", + "71eae9be60b7fb6c", + "3d149fe7c08bfc58", + "1d9ae732c5dec242", + "6bdca9f13e19572e", + "d6ade0a63d0f9130", + "da1139ecdc3512da", + "637657613004dbbd", + "29fb07d8f74e734b", + "075ddd00e07fa7ed", + "ce92195cf4a19084", + "68875c05e00bd1aa", + "61e862ed07d3cb03", + "cdbbbad526f0aa13", + "1d1df0fb541e49b8", + "5d4aae047020163b", + "c67724216ed5a318", + "10229cdd92c1989f", + "4dbb108406acb1a5", + "9292ab2134e8a712", + "84501b4439b66fd9", + "35efe13fa9e93a68", + "5c5431cca026fb3e", + "e3a743ef04ecfb64", + "89d1f7ab8b77e41d", + "70908a4cfdc5585c", + "8035f242b12cec6d", + "67240adc18be338d", + "3e08a27e9d9c9b1e", + "5eb91b620bbf5bfc", + "7e92dcac20e009a5", + "49e6665ee56d3bfc", + "54fada9909a1edb3", + "8ae9a7fbbb50b6ed", + "61e7ec1c62704f39", + "f3bedaa5ffeeadb1", + "a3d152e25f8ce6c9", + "de2ce0c1b72d5c90", + "92ae986997c1d45c", + "29cc495628e76ffa", + "83bb8420c3cccf85", + "ddd9691cb063b07d", + "693325ca20a6325b", + "37b94c08f8e25249", + "26c7b173668167d8", + "60ec8738947badb0", + "37fe6036c2ac932b", + "d69309b0a27c929c", + "01e5c6d31f763f56", + "adc6971cdb0613b0", + "ffc97a8fff838fa8", + "9ac6a8bda4fb7dc4", + "e247b50a301ce26c", + "e359ab2e436adf36", + "723ba544431f7874", + "fff289a66cb9ce4b", + "583c12b5bf2423c6", + "98a578f7758bb518", + "10233188161519ee", + "123c94418ddf1ad8", + "0bd651fcbb86f2a9", + "6a46ff1d1a5154b0", + "a5898123ec124b34", + "6fe5828936663aed", + "5506b5232076d409", + "1ea31bb5f6a15995", + "45098872448f5816", + "112fe9f123e277ee", + "09220515a7edf3f9", + "114e39b4da949e6d", + "621335449637e335", + "b82b48b0e915aaac", + "db9267bdc717c801", + "195366bfba7213a5", + "177dee4c1051e612", + "dcd67613012044b3", + "60f0dd83c8f32b8c", + "bff1330c13ccd99b", + "c06c7eb3afa4aea0", + "9ae7edbd0444982d", + "ec21d3c76c5ef65d", + "449c74a92ebb61a4", + "23a85a7f9c07412f", + "7942e835fb18c4ac", + "51c589803c349e44", + "f577607298c4b574", + "8944c4f14bbcaa74", + "f1a73b2232787b26", + "96c904c81aa2a241", + "0ef56cba2362e463", + "b965c3b6a03aefc4", + "35b9be2203d2b9e6", + "da6845ee99cfa275", + "977b04d04def60a7", + "7d3546a4000b5b75", + "82a2bd8fe4ed2087", + "135ca3602f0005bf", + "95774db1eb19b767", + "83020f91ce1aaf68", + "950ba1c7368c397f", + "98642c6f99438194", + "b7209eec8fe7fc58", + "b21be5dd6fdec83b", + "60f08c85a518de1c", + "62eb991153211de6", + "f9d9fe0dd9f7f110", + "017f3cea8275a3e1", + "b644b78aafc367b0", + "808de32aa819821d", + "cc99b5d3806b1e97", + "d07b6e3036abb8f1", + "db28c1d7f201af1a", + "997281c99ac93b3b", + "a596f2412c0f8fe2", + "dc51cf4e51f01086", + "692e91451741ef49", + "d3d257648d4c3105", + "d6d129c94c368155", + "f9a3871824bcd47a", + "d063a93beba2dd65", + "e4dc90de190437b8", + "3feda0b8b5637540", + "8da3e1e348d36e07", + "d905d620783d3768", + "bb509db67c18bc40", + "902dbd430adb77a5", + "cb9a110799bc37f8", + "41bd2924e9afced5", + "a2cbd066271b6b3b", + "6587b26878da24b5", + "5498bfcecf19dfa3", + "2e997ba7beaed34d", + "0208e6b60a0b1a32", + "17bfc3d0570ab3d0", + "c459c68e95d54f87", + "0a6525cdbeb38df6", + "eac2041efb4cf9f7", + "a9f542d8e2be69fd", + "be9ca77bec616fe5", + "09f539aedefa9756", + "86e9da10bc176267", + "ad2351479ff6ee84", + "6b605ff743c859d0", + "59820117bb7fa240", + "f5a53fa6d2c3d9b2", + "9cc6c08cbaa694f2", + "5ee9898421c06859", + "93f14cad1cb201fa", + "183a5cd3e2aaa0ec", + "53bba35f7ea6edc9", + "6eeed6fefb1f0243", + "8b0204e041dd3eb8", + "7386d04b20f354f4", + "a57845715134651a", + "2a91f42389fda90d", + "f7cbb33cc639b5b5", + "b5b8bd0cb8bb7658", + "86462bcec4492bfa", + "75c08d873d0dcc06", + "6d9c2bab8cc8d28c", + "102bbdfcd696d7a9", + "bbc9a3a1d685d5f1", + "ce22219c2049985c", + "e2a45ce49bb2b731", + "21cffadbbb87c205", + "b4768c31e98b2170", + "16b0a0811bc4fe82", + "0ee132dec087b772", + "8bc3658bc958f81f", + "18b7e5bdf459ca13", + "f2516946c797b1a0", + "e56d5d27cb2b3e6f", + "0a6e50445a906052", + "33cada8c95bf55c5", + "ecee66b37d7576db", + "bc25a76284e7c7ce", + "e9c5d02aae54cdc5", + "b3855f3a6c06c01c", + "b24a14935edd51ad", + "69b877d99c360f97", + "f731aaaab07bdca2", + "7980d204dbc3ecee", + "8c6c4b4606608dfe", + "6e9496b7254e02e2", + "aadbb9992a672a1b", + "ca1fa7c3f26bb3c5", + "27ccec0740bb7915", + "ee4e169753085913", + "ea4f0c0a7b82f526", + "16f1a518cd842fcb", + "83c1ee8ab43ea00a", + "76d52f916b4222b1", + "4d2a177384b23fd6", + "43a4ca62d0c0b0da", + "e23f9c02819688f6", + "16b24ff1bf3c079b", + "cd72fc2b62780a43", + "6eacef3b2f976ce9", + "fbdaf22c4f50391c", + "107c348f4ba0a41c", + "fbe41655a05a01ff", + "39c07ebbca541145", + "173704f0d2b8a02f", + "096b05114ee5e09f", + "a289b9d43310182b", + "cc6918e95b803734", + "5f84cd800ee96468", + "07ce7e4844f62c5c", + "22ede463dbf1a105", + "8ee7145a1b48b578", + "85fa17a19251a2ea", + "58ece75f7f2b5099", + "420876ec1e5ba657", + "2d0b4100c3ead88a", + "42f53e51476f1a3c", + "96c685a3cdd9ed59", + "eacc2cf0173eb438", + "9938be53082a02e6", + "d75ff490c057793c", + "7dfcefd1ffeaa941", + "131ec924c42ef90b", + "d3323745a51a604d", + "2eee0fdd5cdaf565", + "bdf8d5624e230986", + "7096b882b315807a", + "84b1520706c3d784", + "b49efd086851f84d", + "e56e14e8cbf89e3c", + "3183669bf78db43b", + "c28289a45fc012a6", + "a14f221cf1761a8c", + "e7a197d6aefb66df", + "0349c893b81f7c1a", + "84ff17001564ea47", + "b1401488f998c1f0", + "e367ecaf05dacedb", + "6f90b0bf6bbe884b", + "4abc83fee59586b2", + "e64266a9c13fc74e", + "37c8d23c44b95833", + "5edefd19948ad581", + "932cef2b9698dde8", + "158950ed4bc35274", + "4dc023ea2e3c4115", + "8bbbd89d09bb82ab", + "ac4687f4b15507b6", + "3392558f04e04663", + "64e80c255d51ff37", + "78ef46252d580af0", + "4b5521f85ab3fff7", + "6c3e0fe95400f1f5", + "988394e53c8e52d9", + "983d431922822165", + "0319b03323f257d1", + "3a3b0a9113835307", + "91108cfcecb98297", + "b883e389d34b1bb1", + "af58ba642f4fc3d2", + "b0fd694500bfffab", + "d7501476836cf315", + "1cc05dbb992431b9", + "d4a1feebc1e1cb5d", + "ea316d47c03f9fdf", + "781c243d0021d4c5", + "aa3f466daa34d3ff", + "568c473e6e5e95c7", + "cdecc0eedcf05d2c", + "d3987535a95fbffd", + "f4da9e13977580f4", + "c483baec90d76587", + "6becc4c21730d391", + "7c04881c9a46b337", + "a93edea77d9f3338", + "c1e14f36f031a342", + "b12b4e4eacad1497", + "fd583001a87023fd", + "63e55780fa1766de", + "58ef8ca403038960", + "b9a6922899b66b8b", + "0ff82b1cc24b3105", + "0c9e215997d96f44", + "f0bd6a29eee94a2c", + "4f8db13958be0f75", + "c4f5a7748428e7d1", + "adc9df2489ce090a", + "e3231fac88242d81", + "c62b878b1ac43df9", + "a6c1dd30884a197c", + "5469d2b3e5fb8323", + "f588fbe6fd36bc6b", + "04c5a5a66fb7f89a", + "34ed76f0c32d9e93", + "6dfa24c82e89436b", + "d12a50d33b1c4253", + "ab88698a0c6ee1bb", + "13e5b88ad3becb77", + "cce214002aca738f", + "b6c3a9d53b9e784a", + "b1b04d862f4c32af", + "dbbb56b48acbaf22", + "adb678f7143014a0", + "e0999b85d2f77733", + "438b59086f6ebd64", + "6700028a309c3f7f", + "9337ab9573b366c1", + "4c0e2794cb729825", + "6f480809fc73b635", + "2760aeedce0b79db", + "bec58e970e8cd6c8", + "d3420c2bf8974a5d", + "57e929939f1fd1c0", + "65a47f93451f13de", + "b168db2651cbc1d6", + "75609423e1f40c07", + "0d4a95a8b52b787b", + "296618f10b019d9c", + "fce7e01f2cd0cac0", + "36641ce7192b5f92", + "4d21561351ecd00a", + "06b24bc51eb69188", + "dbe20d7c305b24eb", + "dbcf1311b3d95cdc", + "51a4bea9a77a55cf", + "9f400f89490fb8d5", + "55669bcb3a9e09c0", + "6337d3dcc5362dc6", + "55f9f2fedfc7f069", + "3d756c83419bba28", + "8f3037c89c498e88", + "2be10f342e68a89e", + "e501cafa3b025fb9", + "42bc15eb26cf2eec", + "05822f92f304fcd6", + "44becd7e128f4ba0", + "696c193b3d13924e", + "c5bc8b961a199646", + "2a07b85f61ac08b0", + "5cfe68ed449e0478", + "02c6fbcd90dcf05a", + "4285d9b580f8a2a6", + "16513a1bff79b46a", + "a334142059d865a2", + "a67919286833a7df", + "2078424be58e4db1", + "0f684159ad31f883", + "a76054a284f0ce61", + "8870ddf70bf57890", + "15b121492722bffb", + "a781ac2579798523", + "bf1338e9abe85c39", + "86f1afdbbd29e48c", + "afcf1165734847de", + "f569fb2a6ddae452", + "882a9d016c7e1f36", + "b584e424a8b3dabe", + "d80c26c2484dc87d", + "30e9ce0969941ffb", + "8cfcf8dc64485a74", + "d1df1bf926a69ca9", + "f621ca835cf98a6d", + "3519c14e4114d93d", + "db28c203846775d8", + "2d046178aa118fff", + "40fc6ad0aed92913", + "48d897f17754c7ce", + "b5968cd571a14cf3", + "31fd07069aebf013", + "c18dc9f29967fe7f", + "afe441f40a42865e", + "f8a3fde3a557f44d", + "c8e381408b191a02", + "f9e9366287e79e4d", + "cc64f3b8666b17bf", + "e535b7a4343399e7", + "01ce6f422bec823d", + "e64eedbb40d09a50", + "ca88a34b0fcc5b0e", + "bd346dd3e46dff68", + "966a03f9f111aa35", + "b95f2b009d70fad1", + "337ece90615ed69d", + "f14484e630df1dec", + "10af4a6378d0fb7a", + "5cec11774890d61e", + "b99c572f58cfb19e", + "b6c15035f759ec50", + "975dae6e02765a90", + "c96ac4c4015414ad", + "ea1b5ef156502d98", + "082380caaa153ee6", + "1ff08f79cb89c1de", + "87abd998f3b7e26f", + "13d2556ef392ed41", + "00a2b91e5b58cef1", + "cc5a3d2a52f99ab3", + "7a5a4a9578503a55", + "59b026d889aa0309", + "157f811757a0b29e", + "8825e2c99223c084", + "ed511caf8988dc45", + "04cb360502da6492", + "d317c9c2e53ce427", + "f9fb8170cecedc23", + "e3129aad84d5c40c", + "bb21cecb8bbfdf63", + "46459cfb9c1045d8", + "102707a8c4d8a1b0", + "2bd9661657e78278", + "a191b2da409f3518", + "62313881916550ca", + "e5863682239df24b", + "c572cacbeb840621", + "e93cb94a06c6957e", + "cc5e2f58b681e555", + "259e0ab1c4b690cf", + "fb2cfdfef4f93536", + "abef2b316c54f9e7", + "7f02a580f5219377", + "819d92a6492415d3", + "dd4cfd03d9b3f25f", + "996691f0e03e303d", + "5e4e16da9d0630dd", + "5162627872566ada", + "a465a767ceb7fd30", + "cce1a95634c6972b", + "7a26d92281df3e8e", + "109eb2158ca6a2f9", + "743318f3a2b4e281", + "06998c36415036aa", + "a49706f436a6873d", + "75ce95f62fc65fac", + "43d9cb9d3fe30ff2", + "4871a084d74a4bc0", + "8ade0910a89717a8", + "00b539d629e60965", + "74d685c3723bdf92", + "d08b5ddedd34448d", + "6999db84bd629b32", + "f7941427d5fa40ea", + "92c7355f5ea28e52", + "0f7ce013d15f6ff5", + "7b63c31cfd0eb65f", + "15fae84c1e6c6c57", + "4eb33712d4b16f43", + "67e75c6c6538110e", + "8817adf6eaf4f06c", + "97f302708066dc04", + "02cb3c385865ad2c", + "c6c291cd525225b7", + "f2e7c6dfd6a431c8", + "8820852f4a1163b3", + "ec8a2a25cdbda634", + "5bfbc4a6cbab9eb1", + "491c766e35ae9f6e", + "3f22bd3eaef5d64d", + "c497da3fdb2f5027", + "b6e60b9025d116e4", + "e89a1a7817dae7e4", + "15ae64b85b3e28d7", + "7e8763f553e43d12", + "691b730a7bbb28b0", + "8d92202f35f15712", + "d34363168b1590bd", + "e6aaffd588f2832e", + "57e108dcd6ee072b", + "6529877d9799822a", + "384dca297bd4f1ca", + "b764f5d5391e431e", + "b5b8de155e4e40e6", + "d4647f5217d84ba3", + "dbd62ef3bd952bad", + "8dcdbce3262bfcd6", + "835211383d281d89", + "b7132fe23fbdf9e1", + "de50abaf95e79d34", + "c621d1371b314913", + "6ff89f3123a6a98f", + "5caf1c90ec7001b7", + "596a0fd33f70da59", + "40fa0a4cb5193977", + "d3c2a59a2cffa30c", + "9ba1fa1ae8c3ae67", + "598e2a9978d3c79c", + "08bff656ff61e949", + "0160f6becf137107", + "f67f3737a3b47208", + "c3dc8cfd97735115", + "541da571bac5e279", + "55925bc39df20912", + "af78198822da03a7", + "6f13e8b708b43383", + "5787ff8207fd751d", + "424f6d496746258b", + "2c41f9dfcd5014e6", + "e6dd3d435bd1b82e", + "a29d46ab6efee468", + "f4b4b026e87b5086", + "ae175aa839dbf692", + "97eb8171b51f9aad", + "a2a5ccdf0846ce67", + "aeb79c7e573de0e5", + "42d2e68c29d9eef7", + "f1300cb717ea9335", + "9231adbf747903df", + "e9845b4128afd538", + "e0be32a08b66d264", + "8f7e22207cb3dd8b", + "f68006f6c05e1211", + "40f62cd468a546cc", + "56ebfd7a918891c6", + "cf1c544add4bf860", + "c1754dc2e7d666d8", + "a4d0e44f4527180f", + "3a6a5f536bfbd4d4", + "d07cacdb732c0b0f", + "9973eb31c46f8637", + "925da3b468003e72", + "7c7e48e3aed8bfd4", + "24624a4e60ddedd4", + "3e182053b03f1faf", + "7a059d2754e088a3", + "e6959c2b66202cb8", + "b252ff42ce0b7443", + "5f4cd1504f8b18e3", + "be080cda1829826e", + "0f1b31ac93459d1f", + "ea17788d9a9d12eb", + "5103648897ba1b3a", + "b887b7dcd6f1fdbb", + "1494cff85b3dc2bb", + "2adb03f5294c31c2", + "5cbd46653c3bea1a", + "259df98e7476a843", + "c4e6a655a15144dd", + "9b30bcdbccfd4c68", + "84444ea4fabcc66c", + "01dc77d7f8f94d97", + "3ea02a62856c752a", + "e8adf99d340489ce", + "65af80e5c6538215", + "a00e93f2981ef418", + "ef02bc67ebead2a2", + "9be76c10474e0c80", + "54f754dc95137c47", + "a0621c7e36e9ecb1", + "797ea41cc2e95f08", + "b4fc6a7b6c200b1b", + "fca98ecd003e1b82", + "a5451ba38050a44d", + "58688dfc2c59d3ab", + "f2327302df1aa660", + "30c6777901ae346c", + "6fd73557577cf9ed", + "96b0895b9f712d2f", + "e92ba97c2742e63d", + "fb1fff87c87b82d6", + "122245dd15683e26", + "34506b8fab6dd7c3", + "8cfd0c9f4bcd21b8", + "5b1b55f1997a9439", + "09b07fbbb24e1b04", + "f2eb0ef953f3f147", + "7e9fc9f948e5cb17", + "4893140ae5daadaf", + "f2fd56c195593dd8", + "953e132dd43ec03f", + "1c8567a1b1a6c12a", + "48a41f3f5110db1f", + "032faef0207b715f", + "d2bece2b0eebce5b", + "8d2c50e41a1a0061", + "a9004216663480de", + "8f7cc0f456186028", + "10b17081dd7cad8f", + "6ae356d8abbfdeb9", + "1bcfaefe46abccf0", + "8f81bb35e37f1b4c", + "51768783ab3e67f3", + "b6a96b0dcf6eadf0", + "5b67a83de2aaa93a", + "4c90a96fb1ee09f6", + "cecb6a5d441fba98", + "48fd3d84b1a0bc68", + "00eb3e5c7a4537b1", + "3f188bbf2af01b25", + "7e37ff763a97bc72", + "3ba9f330ce93636b", + "21951e191e55b66e", + "ba521df2c728c5a2", + "ffe4328dcdc6b538", + "ae9574428bd713e6", + "ff0ffb9a32873433", + "56a56d39dbfe3516", + "a925d8f1adc5cfe2", + "7c708f62e3280ec3", + "aa0213dd9f727219", + "3334d22b6559d0e2", + "b2436dcf1c8e5dc5", + "8d68ee5c283ca6b0", + "4057f5dbe4ffa4cd", + "18a515a0fff0c0c3", + "4832a44c0df0bad2", + "c8fc487eec95c21e", + "8782c7c292ef2759", + "36df04ab9c978ab9", + "06e70fa70393d60f", + "559e0907fef71cf6", + "af373587f2c57a24", + "3746512864b941d4", + "1999d4ed82b3188e", + "793075287bf70a81", + "878af76320373ba0", + "ca63425a882729f3", + "95c52fa37ef1eb3b", + "d2d43918e5447c09", + "ab769495f97aab9c", + "2b0342d5b70b7a78", + "ee3a996c18ace68e", + "9d36f8ab4e1b45dc", + "3d8083b1e6c7b336", + "fa8384342c25ec59", + "373e7b1bc9066563", + "26c4912b140fde9f", + "d71d0b082233758e", + "9be0c86102117436", + "f70d226e1598ac41", + "328fee1e52f7ac82", + "70f75a14390522b6", + "1188aebd8681e4fe", + "08408cbc394e7421", + "cc932dbc293df39e", + "d5187fc1cd590073", + "0b0648a9b9a26747", + "0cd406e7c5013b74", + "25c07c5d0a05d86c", + "9e55246befe96fb4", + "a0fff0700364df49", + "f9d96d0eb2bc85cc", + "5eda77c801e2df86", + "ab8cf6af80a5c0e0", + "73dbc35c2b91c4fd", + "8e713eb632acc712", + "56bca69ce012d4c1", + "bc715a45d7584d95", + "bb02df07a0d95e77", + "5423483cd3c3b75a", + "6df8f86449e56c05", + "3e345d3c6002ad55", + "29f2dd90f57ea948", + "ee6718f7a7e3d30b", + "ea330145d3ba738d", + "fb8010928a4bb56b", + "921ecc7c3e5f4884", + "940a18f62c79ffae", + "83b4d7d35a747d1b", + "b15c099c68671895", + "f1b9158cc2b98388", + "8b7145c13f36c324", + "37e0d3cb098458d0", + "080aa6966d757f05", + "ee35d872179a1397", + "6abef2102c86c0fc", + "096ccb6bcb3d2731", + "71d3720cad53b289", + "15ee7651b7ad610c", + "0da9d81c9f563df5", + "6ef82dead8e3437f", + "fd966f807d3ba9fe", + "b804c318fc0c1e9b", + "4750f5114dcaa60d", + "9e7c513fdd35df17", + "b2ba4d07e03b47ea", + "cbe5be4b63677c25", + "53fe111411edfa45", + "87a80325171d8add", + "3f8475de6ea558ac", + "a64e6bf63ab6a299", + "c10a3a62bf9fe782", + "711a7b593ed41b02", + "5a1764ef77908457", + "6c2fc8308efeccb4", + "1ac8de53c0894d18", + "bf0f48ffd0dff903", + "a851c3b4377f4923", + "f96bfb09bb1bd640", + "fd3f988b7a505c45", + "84c8bf1f809380bc", + "c16f3dafab92df61", + "55e85aac6594c2fc", + "6cef8442092da1da", + "1172a88e009dc248", + "af4c9b6e93062fde", + "96106c2d31eea8b2", + "c518dd59d12daf17", + "871066e9e2eaba52", + "acee5f619f09bf79", + "0fe0bb1054f49b7f" +] \ No newline at end of file diff --git a/src/sec_certs/data/reference_annotations/split/train.json b/src/sec_certs/data/reference_annotations/split/train.json new file mode 100644 index 00000000..b2e65750 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/split/train.json @@ -0,0 +1,466 @@ +[ + "412316e54e63beef", + "ce3ee1fa409676b3", + "cb862d0020c32547", + "bda89f3b2429ab6a", + "388653542ec9fb6c", + "7f1f7bffe46d4c85", + "0f1bd208c27d202c", + "b8370ceababbca40", + "12e20466a0f08342", + "9b8b9b7fcd97bc60", + "32e083734907ad15", + "294ef6cb28408ec9", + "f17fe9ceea628f62", + "6f38b9bce48590e3", + "b61363e51ed90c7a", + "9328c028356ab522", + "11687f8008b8198d", + "12c3b8b612c581cb", + "db3b58c1836b220e", + "f87b5bc03e18d010", + "a3de319045809828", + "ed22d4c0f09c3e3a", + "09b1192474bc8b27", + "d4e1874210b91dd4", + "c9b0cae8b5784d2d", + "dfe9de5d26e08949", + "97d00a9f198a6e57", + "d75282fdda80b6fd", + "6d41460480570d22", + "a7666a5157a31d32", + "5c3806bf79eeab7f", + "d2d0b2521a1f186d", + "6dfe436482047387", + "7e426c62da58175b", + "9dbb21687510991c", + "2b2dd14fee46498d", + "cf32640f396edb87", + "4d5cb991d2675a6a", + "751a324c3f87f58a", + "8bb25a14b21fcce6", + "243291b6c7ecde61", + "13a92431f9683226", + "7f4d3b659fc17c09", + "dba4b5a166f9456a", + "768dda4099b6f2d9", + "668b5c95eb0229c1", + "1b797b27b981aafe", + "17b5257b3755a07c", + "1c34d31598257da8", + "775b1fdecba5cb81", + "e0265f0fb8e196c0", + "161216fcc9eb8d34", + "7574381766e293c7", + "9529eb793550093c", + "4960693ee45f2ae6", + "cacdfdafce47678c", + "60f3bd10ee9be85b", + "4f5f41ecf7517e63", + "c7fc146c5ccf4821", + "2477ebdeb9bcf4aa", + "6ca1063d82ab795a", + "d2a4b8cb9ae7fe8f", + "c2024b43eeaa96de", + "a6fe8fe0aaf2fa92", + "815d67f8292ad7c3", + "19c60923e31777e7", + "97d5f9e6e0a11e37", + "c39e6ef4060594e2", + "a76d1c0d9964d583", + "a52f807db0eb7fdb", + "6e4de0c194952a21", + "317713948b4473b3", + "73e4f65789159996", + "54375fc2889bdfb4", + "2731d8ddba404fad", + "617810f97f3f0b4f", + "e8e5f42a4e4005c7", + "03aded94fb04c62e", + "1fb1564dfb0f0b04", + "e7e366f3619b5953", + "cd43abdf668dbb23", + "1fa2284c3dcf136a", + "c96343baf608174d", + "c32378a010479b33", + "99223aca5d9eb3b3", + "678b2b51e429444a", + "727bce62137f97b7", + "62bf5de838cd3a7c", + "732271484650d27a", + "b42009eb34746731", + "2b944ab6ce8fe8e1", + "8ff91b407ccd4b18", + "8fbe28bc70a821bd", + "86255c61e33c2caf", + "49bf2cf97dd4ca29", + "9776cff7d7ead7bb", + "69347104bc7e740a", + "818f62b997d143e3", + "bc17c54936484a90", + "824a8da5564344a3", + "fe223a9f01625d62", + "4c3282bd1640fcbc", + "ee1c6dd97918d74a", + "ace99e269d2a1bd9", + "4f3bf2d41fc130df", + "ca5da2fe138af656", + "41fb9717f70d3037", + "de9a98c311683ef3", + "bc260c38202644f4", + "30120e4f3aa2f30a", + "e66a9b22df7fe32d", + "559a5c3c4c23a9d3", + "3515801dee00995f", + "f20be2311365981c", + "e10bc3422afaec8a", + "5734ff91ea85611b", + "dabbb27e6faa13d5", + "7a8d4ed693d443d7", + "d94b13d95fd1c12a", + "e2e705cabd42e40e", + "b434dea9be2868db", + "e9e12b044b495c6f", + "a07548a64fac5794", + "5df7b3241b307f8b", + "18e68a485e648ddc", + "01e8805514b4ef67", + "13d95239226aa537", + "3f87aa90c0aaf83b", + "f2dc875bd399a806", + "a83fe0ea2f391ee8", + "7793c695818500d2", + "a38fbb576027f6c6", + "2f418fb3ced3820a", + "b5b0e3205923e723", + "627bfd69c2a831cf", + "e0b122da55f1f002", + "f6579adcbf5faa99", + "b33eaac3aae2f1e7", + "e6e8add5e4db2d9d", + "b0b7b073ca2dfe5f", + "fa67fbbcfc0b9442", + "af61a31e3fd0d6f0", + "fe445dc1c13738d6", + "0d08a211bd29f963", + "f02cf5078e4548e0", + "a276bd58a15d5be6", + "9d0a023ecd4ad7c5", + "0ff9337564f4f742", + "fbb507bc72376d36", + "6ddd0d9c37f27706", + "6a2323c5267ae0b7", + "744a7a202d909323", + "714559f7e1bf956c", + "c92a196d0408a5ff", + "f869956d14ea1694", + "34370b67b5e675c3", + "388b6899fb38b762", + "dd81eae9478576f3", + "8975cbb58ab67846", + "055985699aee1e09", + "1416afc51d8ce022", + "eea87713e11a55c2", + "44f1f74c09e634cb", + "598b6c745cdcd404", + "e9b7e07dc5598afe", + "0b3e1cdf3ef9413d", + "610e118d804e4076", + "fb9ec9e846ea4e87", + "aa096ffde94b953b", + "887afbb0843425b6", + "e2275acc4f1c2d0d", + "b3b7ac7aae87793d", + "95b2a54b81b0bc6c", + "77acb3a4a3f58e0b", + "ecfb020c517d37c1", + "9302a95ca3efba5c", + "ced6075dd1813122", + "ab3af998dff7a2ef", + "0f3900cdcd0c7f3e", + "d699ed2b1adc6be4", + "1e40bd733acd56bc", + "aea11fd4d2a7709d", + "cf7705e45048032e", + "1d89cece70c6cb64", + "f73d08f8cdfe276d", + "cfdacd53c732343c", + "9a94525307a1b13d", + "543d800fd468f0fc", + "25eade3365f92578", + "e886bfa3a4f4ffd4", + "b8437dbfeeaf0f04", + "05c03233d710cf54", + "18bb29b147e80caa", + "10107b83e7393c50", + "65ab2fd5467ef78d", + "10868774b23c5730", + "253e4d53f952a882", + "a47713a44e4c656b", + "f2e00a1fee27bdd7", + "4783477a22587a6e", + "15d935e36152eb15", + "4a6100f1ddaef93b", + "680aeb0a20a9fed3", + "14df54305cad83b4", + "188d3cda4e6f2d5d", + "fc373293fb7c56ac", + "79f48541164e16ff", + "e9fc2f8b110e71a0", + "ce14eb1b2318f054", + "12fd6b1b6c4aecf3", + "5f4b39feb2a82dac", + "0f040d1ec658b8a5", + "09be49631855ae19", + "c8ff17063166050d", + "76b289d10702db4d", + "eac7be8886cb755b", + "84c868322995b179", + "a515a81bc4be3cd2", + "b2bab86575d21b44", + "0eff4063e68d7cd6", + "32431779b1c54c0a", + "c118ee1ff2d1bede", + "1b0a6ecee1f830fc", + "cd052c606f2f9124", + "5deb0179858d78a1", + "dd1472476dfe4a8b", + "1a191f3535ccaea3", + "a9d336b90e8b94a2", + "6c222ea916b4f5e1", + "c5a60dbdb668bc10", + "f3a461d6bf980196", + "977477f3f0bf1bfc", + "1ee7ecee9e7e131c", + "d1a87288683080c4", + "185aa4f76ff15d62", + "23ce630f1ab642b8", + "7e9dd6cb86b58f95", + "b30ef2c1aaec9236", + "a154a01ff4cf04c7", + "79c729d80cb73939", + "a9639521d8a5ab09", + "f32a1e7e2843c096", + "8c21cafc192eeea5", + "59f15180eddee8b2", + "2ebb5bdbe7e00a78", + "bd1a21ea3d047c14", + "dbf15cc4f389196f", + "98f6f9dbb1540a31", + "cb203cf5d91b1ae3", + "f921d39906e6d93d", + "6c14f733fe7aa28e", + "f71fca5fc684df8b", + "7e82a1ceb29e4019", + "54273dd266fce692", + "536aac44b608f951", + "921e042759b30033", + "0167c92c0d8c8b47", + "90581f56f6846be5", + "a91ca8347f800ee7", + "dc278166b5d14512", + "90c290ed4ded479d", + "cee70006132a6f46", + "1357546ec01b1a65", + "5a66027d34aafc52", + "2c4caf71ca3735a8", + "0baac042dac42abc", + "54dd28994604ee35", + "60b45a1573fc53e9", + "6a999675c9422dfb", + "0d164fb0d72dfa00", + "b24f997d9358b324", + "df6728b8420ab3b4", + "d50ab8fd0c085fb6", + "9ae8bef18938d2df", + "19d14dec2d11fd96", + "f0ce40dfe333e918", + "3da6e0f0f97b3d2f", + "0d730f18ae9d5694", + "ecba01acba8df2ec", + "17a23970b35a4f44", + "1b8705a0486b3be1", + "1ca9b79a98ef45fa", + "66e47f5d28e2f95b", + "5f1bc82727f54376", + "9743174f44c5e2c9", + "ec84060a194fd9cc", + "845bb039719ac5d8", + "ea6d47427a2bb349", + "27d11629261d8806", + "6a5cb89e80f86ce9", + "29b0321ec6b75ebd", + "d355c778fcfee3a9", + "0ce083454814c1ba", + "c06d86db280f9b2b", + "b0e95af7c706ffa0", + "b9a973281ea252b3", + "d0705c9e6fbaeba3", + "a3210e75e10a3ba8", + "61832cb4291c343f", + "22dd52fbce13f27e", + "985ffde955eee01a", + "882df28aee338320", + "b23babf61349db1a", + "1c9bd4db3d8388f7", + "1ff5d648a625f3c7", + "691119fe8a2f8e1c", + "c30de3192d2e8ec2", + "5bf4e37462a1b715", + "de23f9f77f9b49f0", + "dadd7dd2d4247627", + "36ed04f4b45e3ab9", + "6b249c38b49ec9f0", + "a69ec271f8651d1b", + "7f9eb1b8217d15ad", + "6b003152c5459126", + "6eb6b29c45f6355f", + "502645d74f607b06", + "6d6ade44dcc497dd", + "3ebe69af702fa116", + "29c65bb00fb9aa6c", + "b4a2999b439b1ba8", + "3b0c5666e54a2394", + "8bdb610131555c12", + "b10ee0bb166cfd86", + "e676143c80802a59", + "15d68159595eae09", + "06b1a65aa87ffa78", + "5ee0fb9826fec164", + "2c15dfc106ebf8d8", + "bab97726875c0f14", + "a867d281d34d34b7", + "eeb2e28f54484dc9", + "d753660b3c2bdd84", + "5f7760ca5dd8b8ef", + "8ba22f6c9651edc3", + "9650ee69410c68ec", + "608fe49756dc68cc", + "e7bfc160fd3c0d5d", + "335800e03030fb93", + "446aa68e0c4c5083", + "1e58fd6b31d72e2a", + "4c1ccde478ce063a", + "5cc6ace5ef0d7525", + "09b17cb9b3c8b1bb", + "bbf0ceb1e0e4c6ee", + "c78ba3edcbd57788", + "ead1076787fdd7e4", + "8ecc2cf269498bee", + "d6d6b7c041820ab8", + "d837b4fde544a9f2", + "3ef6d59aeaa6d26e", + "fca2af05ee78ac5c", + "cf5cfcae47e5e3a1", + "baa8b8c5f52810af", + "3f45ee333ce457bb", + "7e24a5a087db6814", + "30f71b100c5cebae", + "9a22a2bb323cfd0f", + "84a75b6fc33b669e", + "21d71bac070403e0", + "0d4385899adc0781", + "05726637bd47a762", + "7a9f073396eaec67", + "5cdef03a3004a6ff", + "7d8dc0575933f142", + "890f788a91fc2fa1", + "5682e68143deea61", + "6dc47e3f5d35aedd", + "16bdbde359584f99", + "0bf7a19b22163465", + "bdd16afe92166295", + "9ad46ff913bb805f", + "b07adba37aa1b7e2", + "3cdca4cc05854ade", + "41148b8edaf1106e", + "85054d3f47bee56c", + "aabc3155ba9aec95", + "dbaf9789160ca414", + "463ecd64b7506048", + "cef1c9957f4ff612", + "1ee8a6a4d8f26d65", + "a92a031f28de3544", + "c1a87780d88d85d9", + "659f501baa5997da", + "95c69029642bdb41", + "e10d907c64d75e3a", + "4fd2ed9e6088176e", + "1eabf957a9401fab", + "cf8ac5972027dda5", + "623dee827372d496", + "f041e5b526e79ef4", + "0832672073492f50", + "0213bdb5ebb8dc19", + "d49988efd778ca9d", + "c301c38902477230", + "376b8eb06fc0b40a", + "037577fc2019fcfa", + "14e053dfe9057aa8", + "28e70f46ba4394dd", + "b9482c3bd09d03dd", + "d2b0cb5a911b8ef8", + "05a8a84b60a30ceb", + "63d0bacbab451d2d", + "5f3c4c189362c2e5", + "c7d162e62f174cd5", + "1f9f4b843070fc1b", + "850f5b2c312299d5", + "b9d8c4cb5cf8a4d6", + "f6147b5c0a147123", + "3db023c27932ad93", + "42a062fe7f1ab253", + "a904d6cb272e8533", + "e86db251be2c8a60", + "5e5d4b87195388c7", + "9ef6037ebd29a9be", + "242fdec5d0ed5010", + "0bd759633c0ceec6", + "6f8d7a6a1dea6a3a", + "a55ace0305e39654", + "19f06ccd7136a2b4", + "2e2d56abe9dd6e22", + "c385c0ffaa17bd29", + "c38734a9eeea0ff8", + "3897d8d16ba333e8", + "d7467eab308fb4a6", + "4761b0a9b046cc66", + "0ac0120f667a8dcf", + "ed10036ddfbda912", + "00efeb17bcaafce6", + "61bff064f593ecc0", + "c3549567b1ad7bdf", + "371dca18821f7714", + "a6cdbfd0a7bef417", + "c175537338906951", + "c7f6cb518fb19151", + "6291bcc06184d125", + "06e505abb8dad1b8", + "82b7b915b7af9c17", + "725b6b95cd47b601", + "4bede5d2af4cb11f", + "600e2a1d0f05afd9", + "03281d5d8a77be56", + "27b4366a0f1fcdef", + "42799014c183e1b3", + "9c6440ecec046cc0", + "b26c9f300c14239d", + "366786e98797ebf3", + "454f266801ae2ae0", + "fdbbd2b53cb2e1e3", + "76610bf4610246c7", + "09963e79b21cde12", + "457fcec1ad31841f", + "6962629b904be54d", + "a7d6e1aeb5e009f0", + "2674228d88c4f3d6", + "2bb4685a03857b2c", + "4d776db86abecdda", + "21c3236216f620ff", + "5a12431e371f3ce9", + "2d2ce200fea72359", + "9aa4b513807ea22c", + "4526aa14337a2b0c", + "f659401e91439524", + "dbbf02a1cd0ad33b" +] \ No newline at end of file diff --git a/src/sec_certs/data/reference_annotations/split/valid.json b/src/sec_certs/data/reference_annotations/split/valid.json new file mode 100644 index 00000000..939924f0 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/split/valid.json @@ -0,0 +1,312 @@ +[ + "84d6a370e4f38ddc", + "7482381a8f5cf6b8", + "d5074e4b714ac9e7", + "d11406732e3bea76", + "b865bde82d06ebb3", + "e56a494dd7eca1bd", + "0bae4647efd39e51", + "81273108dd167b98", + "85328590c0d7273f", + "dd000f356b48904d", + "e35db152221bc857", + "d44641bd5ae5278f", + "5bf14a907f9a4c65", + "32d4bf275131aa15", + "78bb780793fbb176", + "e5f9f083c2d03d82", + "7623950728f52a71", + "625b243397001f01", + "59f7dd56d9e1b0a8", + "16abf8ee0697e64f", + "66faa094a9333d1a", + "97cc58c1e41f99b0", + "5f3c9c6bf76ba72d", + "0041baf85c9ca3ec", + "ab4814d2b8c742a0", + "a21d17fd2d8b8edc", + "26351f87a8ee7708", + "4b3f577896c80f8a", + "d96438a7165e5d82", + "50aef770cb4f028f", + "4ea21dc6d8ed8838", + "1d800e909e6141d4", + "f817fca54b13831d", + "22388445fe620ac0", + "f6a89befd13a643f", + "0372b733c28f305a", + "3f2a6d4893c18634", + "350581534e265186", + "88902efe4a3194b0", + "63c1045055ec3b58", + "dbefd21e0104bf7c", + "67c8e242329ba986", + "4d983357fd464bb5", + "340a7fd84bf76abf", + "d314fa7ee445bd1d", + "fd1a78c4cfc3ce73", + "0ea6796582fa5732", + "4489bfc781a82281", + "8927428c96c0b59a", + "cf103f7882a05deb", + "b19a4e8667817aa3", + "7f29aa0291e79ae3", + "7e58bfc14edf68e4", + "d472c3dfb2f2a810", + "c2dc665e589f3592", + "83bc5fbf2b54e83b", + "73a8da91bc0e9ef3", + "f419a1217eb38de5", + "7f776a2d7992cecd", + "c3d0eacd0639efb5", + "83e9484cfd868dd3", + "833ff87d69cff5db", + "60dad220f2ab3639", + "b402cb61be362e5f", + "19c7a1b1a2df87d7", + "a17918ec81a36da6", + "120a49c0aa284f68", + "c18fea56107696a8", + "37a37d1a4986527a", + "234da54efbcf734b", + "459539da9315ccac", + "646d03428a882c1b", + "682ae6d18001b529", + "8c92d4b773f61ca8", + "a6fac58198296194", + "2f0be9733140e2c6", + "edb3056f1838e6ae", + "fc875e0381a28435", + "647d17d44745a532", + "2fb4fdf2a570c366", + "b1ef468658a9a0c1", + "cb20c04ba174a26c", + "629372a8091eafae", + "2f43fe24bd8e91e1", + "c1d88ce9dadd7d2d", + "2f3c913f6a97964d", + "8928e191ee2fd3b9", + "f6934fa14d46e748", + "36b156f7888a84cf", + "afa1af9f34e28040", + "60f347ddb0b38299", + "c7c1215819127a1c", + "ee0a5dc41c7621a8", + "3e077f85d5a6c833", + "b00bb452e7a8176c", + "425f13fb7e94add5", + "8e93586c1ec97514", + "1681e7b35fd0cbf9", + "0fe8df0e85116b61", + "b88f4271f7200c7d", + "be874afd46c1e39a", + "bcac7e3f9f614ec6", + "ce504241f1c04e66", + "9664c0f0ec6401b9", + "92f3743c5ec7a199", + "994ac7d986a99e50", + "96d69242a699323d", + "a5adb726852f5cc5", + "82c717a9a379b75c", + "03fad65e5af65088", + "c63e878cc60a8682", + "2c2244c35d126bfb", + "5a5aa275eb4ec7b5", + "69b15e884cc13c70", + "ec1d79d2314e86c1", + "861434b03ddcac64", + "f5035bed07944920", + "37bae9eaf342542b", + "0dc53ac916660b90", + "a0c38b4389ad7cf7", + "f00a01940be6481d", + "fcfd47d66ec6e37d", + "3cb89a7fc7672364", + "28c5a30fd4464675", + "b3b547454a63153b", + "c37a9b7ae53093d8", + "a7840a951b1de6c2", + "e1b08870ad5117a8", + "798f2870a2d9dcd0", + "bcd8763845d1b292", + "81cbadb9d7a6993c", + "d77bee5cc3a0378d", + "c23bb542201aa511", + "f4d510a39278e687", + "293d35fb3749487b", + "0eefe40a0baa2971", + "0767ea7f695f5166", + "720aab4f782f4c60", + "610581cb76d7590d", + "928a4c5185e10dbd", + "8424207a5053802b", + "3fb4f28492730d2e", + "1e23ae421d7a2d01", + "01bb9c4c07d29bff", + "b8875373ee66e841", + "7537ed41507e461b", + "8d89e926a34fee1b", + "db7627e56b8621b1", + "92aa6b2fcbfc7722", + "d3b779682ee988c8", + "611a1cc1a2d405d3", + "3a3ae16e94bc7aba", + "f654c82bca57cf62", + "2b1f098200b44adf", + "8d4a6cefea9d8f17", + "f357d2d64485b9c9", + "9f0de9801a38d664", + "7ce2c3dbaa47508a", + "0e22fe4e4e58faf4", + "0694786160024e30", + "0f53257eb74bee54", + "94fb86d31446e531", + "031667f4e242da61", + "d696a9daccdb2a9f", + "4c017087976d05eb", + "8037b4b4bc8d26c2", + "73e0b884cd04d9bc", + "c46a9d4c505a95d1", + "13c393fa406a72cb", + "89a05fc32f2f5834", + "d4a83d8a8bd26bab", + "fbc78e2407910a4b", + "37411c359cd8cc6c", + "4ef66341677f73f5", + "48fbeab82953a1e8", + "43109d44111df5d3", + "cfffcf0b8f0516a5", + "cefb1202ed3cd997", + "e0a244a6765fd08f", + "5926a9aa89511b26", + "59ecc4b560f48ba8", + "559b3c20886ed742", + "c8cc75b95f0c8047", + "1e7b90184a127017", + "2d835bff00514238", + "e76ba28e58604c26", + "2cbc543a7c7de8b5", + "aabd0ff0d82fff72", + "37c937488a4647a8", + "5a7c1a87df0e05ed", + "657f8d0cc39bbcdc", + "1816ddecd7432cf9", + "bad93fb821395db2", + "0bb56716a1fbef1e", + "ef9e2e35160ff85e", + "a6dc46792d4e41c8", + "238f8edc5eda1358", + "3357c28c91cc2ed1", + "f05de4db911ec8af", + "c55f7ae8ec2a2b95", + "21f23f5e8c270deb", + "265e576fd3517f91", + "1a27e38bd8df6232", + "cb1947cca068e463", + "c030c8ed94582dce", + "46c90dce23083cfe", + "d185510cb8fd18f9", + "9d69024aca1c4854", + "0d2cab8061464438", + "141b99fabc179550", + "74ceffa7e539fa48", + "6ed3cd21c6a2c9d0", + "dc45c7a6ef17f8b5", + "f0d94de8beabbd84", + "5d13cfce75a45cd1", + "38e7d62918a4f9b6", + "d7c8920acdf478ea", + "1b71508a0007113e", + "077c28a0185d07ab", + "b1f2aa8f6f76b30c", + "8dd850c81b49a1f9", + "09c97bb07b943fc5", + "844703f5c2809777", + "5e731c2ba80088f0", + "2748f920f4df8eaf", + "35482ae218713d54", + "87286bdc06061018", + "0c7ef6c32cbdee47", + "537154d2c4344d48", + "187d237ce11c036e", + "f96206c3f5234d23", + "edfe491576c461c4", + "4edd58bd918ce969", + "fe55e99f4f3dc195", + "6a85bb1d21a35e7d", + "774cb0f28f7900f9", + "df9ccdffca7c28c8", + "f48b3738091c9424", + "b664311c69cff92d", + "22d6578b30f5a227", + "1657515edd068cd6", + "710c8d7f75c0a710", + "4d38305c4a2ee035", + "cae968d4926648bd", + "dc562be6acc8930e", + "d211465bc603f6af", + "0efe2627f1ce8ac2", + "8da263e23f332322", + "bbc41d7d09e40c0c", + "9c662ed62bfd5d2b", + "48e6abb1bafdff36", + "34de7910ace6d2e9", + "e6057ffc5085a192", + "adca05bec28db483", + "011796336c7b94de", + "1754df1f2e3d5adb", + "ef339ad918652d91", + "4fa8af0edccb7ada", + "8532a6232360aff8", + "6d1d34e4c80a3368", + "3983cd9d80386fb4", + "d55eb8c97d71e843", + "000176284683faa2", + "328058c83791df93", + "ebc77980250ee68f", + "cc533c087f06ad0d", + "7b1efef5807fbf94", + "a3c99646acb0bb6f", + "f6317f134f532696", + "67a79bfc84b4071b", + "d7e408fcb899dade", + "8344cee00da693fb", + "fb6b381d9ec012fd", + "bbcfe0aee7ea78ea", + "5dd0014fd38a553d", + "2793414918738c7f", + "983d16512ae92d46", + "1a5f86cb1d942c37", + "991790f44a825778", + "7a12bbde6a73c9af", + "1e545ff455f7ea5d", + "0d94556c4f9d05ad", + "0b36924a005c2ae2", + "6785be05d030779b", + "5140d3b40ac28386", + "7941d384f7b9cde1", + "a67614bd685d7dfc", + "24de96ea505af909", + "38f79f1a48e7844f", + "c24f7253b10c0356", + "13da66c1d4719bbb", + "5c06e4fb1887fdc7", + "2532fe8c7214c485", + "4962c0d8106b8b5e", + "4ef5adf3069403e9", + "e9ed6f2d52911d5c", + "fd34fcd8a64e1253", + "eea7fe4f5d754792", + "a4bbf07046109fc9", + "29964f32c68b0ce8", + "078becead8a70053", + "d0055474970a4f5c", + "24363a67840112f4", + "0656984a0d34dbdd", + "e3b6fe6756441ee1", + "1b20b1629ba7d3a6", + "5cf51622fafaf868", + "ba69597434946260", + "2fdab67916d89c9b", + "14b5a3a574c90154" +] \ No newline at end of file -- cgit v1.3.1 From 55bb6a66260792a9cbdee19649f925528f90f380 Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Thu, 20 Jul 2023 12:15:34 +0200 Subject: update the reference annotation methodology --- src/sec_certs/data/reference_annotations/readme.md | 35 +++++++++++----------- 1 file changed, 17 insertions(+), 18 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md index c17c55dd..b3a240a9 100644 --- a/src/sec_certs/data/reference_annotations/readme.md +++ b/src/sec_certs/data/reference_annotations/readme.md @@ -9,14 +9,19 @@ This folder contains data and the methodology (presented below) related to learn After manually inspecting random certificates, we have identified the following reference meanings: -- **Component used**: The referenced certificate is a component used in the examined certificate (e.g., IC used by a smartcard). Some evaluation results were likely shared/re-used. This is a specific case of *evaluation reused* label -- **Component shared**: The referenced certificate shares some components with the examined certificate. Some evaluation results were likely shared/re-used. This is a specific case of *evaluation reused* label. -- **Evaluation reused**: The evaluation results of the referenced certificate were used for evaluation of the examined certificate, due to reasons that could not be resolved as *component used* nor *component shared*. -- **Recertification**: The examined certificate is a re-certification of the referenced certificate. -- **Previous version**: The product in the referenced certificate is a previous version of the product in the examined certificate and the recertification is not explicitly mentioned. -- **On platform**: The examined certificate runs on a platform that is certified in the referenced certificate. -- **Self**: The referenced certificate is the same as the examined certificate. +- **Component used**: The referenced certificate is a component used in the examined certificate (e.g., IC used by a smartcard). Some evaluation results were likely shared/re-used. +- **Component shared**: The referenced certificate shares some components with the examined certificate. Some evaluation results were likely shared/re-used. +- **Evaluation reused**: The evaluation results of the referenced certificate were used for evaluation of the examined certificate, due to reasons that could not be resolved. +- **Re-certification**: The examined certificate is a re-certification of the referenced certificate. +- **Previous version**: The product in the referenced certificate is a previous version of the product in the examined certificate and the re-certification is not explicitly mentioned. +- **Unknown**: The annotator could not assign any of the previous contexts. +These can be further merged into the following super-categories: + +- **Component used or shared** +- **Previous version of re-certification** +- **Evaluation reused** - these cases should be manually visited +- **Unknown** ### Reference classification methodology @@ -25,17 +30,11 @@ After manually inspecting random certificates, we have identified the following 1. Inspect random certificates (>100) and capture the observed relations into reference taxonomy 2. Split all certificates for which we register a direct outgoing reference in either security target or certification report into `train/valid/test` splits in `30/20/50` fashion. - See [split](split/) -3. Choose all samples from test set, random 100 samples from train set and random 100 samples from validation set for manual annotations. Store those into [manual_annotations](manual_annotations/) -4. Label all references in [manual_annotations](manual_annotations/) as follows: - - The reference meaning is recovered based on certification report and security target pdf. - - The annotator should visit certification report first. If data in certification report is ambigous or does not allow precise annotation, the annotator should further visit the security target pdf. - - The annotator should set `None` label on instances that he/she was unable to decide. - - The annotator should label the `source` as: - - `report` if he/she decided purely based on the data from certification report - - `target` if he/she decided purely based on the data from security target - - `report+target` if both artifacts were needed for resolution - - Note that this label is only internal, not intended to be used for research - - The label should be in lowercase letters with underscore, e.g., `evaluation_reused`. +3. Label all references in as follows: + - Extract the text segments (using `ReferenceSegmentExtractor`) related to the references + - Use label-studio `Natural Language Processing -> text classification` setup to assign 1 label to each of the references. + - All text segments both from certification report and security target are displayed for the given instance + - The instances labeled with `Unknown` are re-visited and labeled after manual inspection of both certification report and security target pdfs - Artifacts of the referenced certificate are not examined. - The labeling is done by a pair of co-authors, the inter-annotator agreement is measured with Cohen's Kappa -- cgit v1.3.1 From c6a7e61d7b1c1c0549b07282398b13b0afd45436 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Wed, 26 Jul 2023 15:22:31 +0200 Subject: add train and valid annotations for references --- .../annotator_agreement.ipynb | 4 +- .../manual_annotations/train.csv | 200 ++++++++++---------- .../manual_annotations/valid.csv | 205 +++++++++++---------- 3 files changed, 208 insertions(+), 201 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/notebooks/cc/reference_annotations/annotator_agreement.ipynb b/notebooks/cc/reference_annotations/annotator_agreement.ipynb index cd2dff81..6b941323 100644 --- a/notebooks/cc/reference_annotations/annotator_agreement.ipynb +++ b/notebooks/cc/reference_annotations/annotator_agreement.ipynb @@ -61,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +69,7 @@ "dset = CCDataset.from_json(REPO_ROOT / \"dataset/cc_final_run_may_23/dataset.json\")\n", "conflicting_df[\"report_path\"] = conflicting_df.dgst.map(lambda x: \"file:/\" + str(dset[x].state.report_pdf_path))\n", "conflicting_df[\"st_path\"] = conflicting_df.dgst.map(lambda x: \"file:/\" + str(dset[x].state.st_pdf_path))\n", - "conflicting_df.loc[:, [\"referenced_cert_id\", \"label_adam\", \"label_jano\", \"comment\", \"report_path\", \"st_path\"]].to_csv(REF_ANNOTATIONS_PATH / \"conflicting.csv\", index=False)" + "conflicting_df.loc[:, [\"dgst\", \"referenced_cert_id\", \"label_adam\", \"label_jano\", \"comment\", \"report_path\", \"st_path\"]].to_csv(REF_ANNOTATIONS_PATH / \"conflicting.csv\", index=False)" ] } ], diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv index b29c609e..ca8f2774 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv @@ -1,101 +1,101 @@ dgst,referenced_cert_id,source,label,comment -e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,None,None,None -0bf7a19b22163465,ANSSI-CC-2016/44,None,None,None -ab3af998dff7a2ef,ANSSI-CC-2017/47,None,None,None -8ba22f6c9651edc3,ANSSI-CC-2013/55,None,None,None -680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,None,None,None -3515801dee00995f,BSI-DSZ-CC-0447-2008,None,None,None -6a999675c9422dfb,BSI-DSZ-CC-0353-2006,None,None,None -12e20466a0f08342,ANSSI-CC-2016/79,None,None,None -d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,None,None,None -921e042759b30033,BSI-DSZ-CC-0978-2016,None,None,None -00efeb17bcaafce6,ANSSI-CC-2009/34,None,None,None -c175537338906951,ANSSI-CC-2016/70,None,None,None -fb9ec9e846ea4e87,ANSSI-CC-2017/47,None,None,None -ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,None,None,None -f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,None,None,None -cacdfdafce47678c,ANSSI-CC-2014/59,None,None,None -dba4b5a166f9456a,ANSSI-CC-2014/76,None,None,None -aa096ffde94b953b,BSI-DSZ-CC-0411-2007,None,None,None -b42009eb34746731,ANSSI-CC-2021/50,None,None,None -463ecd64b7506048,ANSSI-CC-2018/52,None,None,None -c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,None,None,None -c96343baf608174d,BSI-DSZ-CC-0645-2010,None,None,None -0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,None,None,None -2c15dfc106ebf8d8,ANSSI-CC-2012/24,None,None,None -e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,None -ee1c6dd97918d74a,ANSSI-CC-2012/71,None,None,None -e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,None,None,None -845bb039719ac5d8,ANSSI-CC-2010/40,None,None,None -af61a31e3fd0d6f0,ANSSI-CC-2010/02,None,None,None -d49988efd778ca9d,ANSSI-CC-2010/02,None,None,None -54273dd266fce692,ANSSI-CC-2015/15,None,None,None -4bede5d2af4cb11f,ANSSI-CC-2019/59,None,None,None -d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,None,None,None -a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,None,None,None -f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,None,None,None -8bdb610131555c12,BSI-DSZ-CC-0645-2010,None,None,None -b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,None -371dca18821f7714,BSI-DSZ-CC-0945-2017,None,None,None -f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,None,None,None -e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,None,None,None -4f5f41ecf7517e63,ANSSI-CC-2016/80,None,None,None -a867d281d34d34b7,BSI-DSZ-CC-0976-2015,None,None,None -16bdbde359584f99,ANSSI-CC-2017/61,None,None,None -34370b67b5e675c3,ANSSI-CC-2013/47,None,None,None -a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,None,None,None -0bf7a19b22163465,ANSSI-CC-2015/80,None,None,None -1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,None,None,None -13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,None,None,None -f869956d14ea1694,ANSSI-CC-2012/68,None,None,None -037577fc2019fcfa,ANSSI-CC-2021/49,None,None,None -17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,None,None,None -5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,None,None,None -1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,None,None,None -28e70f46ba4394dd,ANSSI-CC-2012/08,None,None,None -61832cb4291c343f,BSI-DSZ-CC-0955-2016,None,None,None -42799014c183e1b3,BSI-DSZ-CC-0312-2005,None,None,None -627bfd69c2a831cf,ANSSI-CC-2014/37,None,None,None -aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,None,None,None -05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,None,None,None -9529eb793550093c,ANSSI-CC-2018/40,None,None,None -7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,None,None,None -ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,None,None,None -ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,None,None,None -ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,None,None,None -30f71b100c5cebae,ANSSI-CC-2015/15,None,None,None -6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,None,None,None -06e505abb8dad1b8,ANSSI-CC-2017/54,None,None,None -317713948b4473b3,BSI-DSZ-CC-0798-2012,None,None,None -055985699aee1e09,ANSSI-CC-2010/02,None,None,None -b61363e51ed90c7a,ANSSI-CC-2019/12,None,None,None -691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,None,None,None -a07548a64fac5794,BSI-DSZ-CC-0350-2007,None,None,None -19c60923e31777e7,ANSSI-CC-2017/49,None,None,None -f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,None,None,None -c96343baf608174d,BSI-DSZ-CC-0750-2011,None,None,None -42799014c183e1b3,BSI-DSZ-CC-0429-2007,None,None,None -3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,None,None,None -5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,None,None,None -4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,None,None,None -c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,None,None,None -ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,None,None,None -30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,None,None,None -c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,None,None,None -ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,None,None,None -bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,None,None,None -97d00a9f198a6e57,ANSSI-CC-2015/66,None,None,None -744a7a202d909323,ANSSI-CC-2017/24,None,None,None -df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,None,None,None -6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,None,None,None -1b8705a0486b3be1,NSCIB-CC-0209053-CR,None,None,None -a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,None,None,None -cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,None,None,None -27d11629261d8806,ANSSI-CC-2014/59,None,None,None -a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,None,None,None -d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,None,None,None -c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,None -84a75b6fc33b669e,ANSSI-CC-2014/08,None,None,None -36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,None,None,None -01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,None,None,None -0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,None,None,None +e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,report,component_used,the previously certified M7892 G12 component +0bf7a19b22163465,ANSSI-CC-2016/44,report,evaluation_reused,unclear whether component used +ab3af998dff7a2ef,ANSSI-CC-2017/47,report,component_used,None +8ba22f6c9651edc3,ANSSI-CC-2013/55,report,component_used,None +680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,report,component_used,None +3515801dee00995f,BSI-DSZ-CC-0447-2008,report,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,report,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +12e20466a0f08342,ANSSI-CC-2016/79,report,evaluation_reused,unclear whether component used +d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,report,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert +921e042759b30033,BSI-DSZ-CC-0978-2016,report,component_used,composition scheme “[COMP]” mentioned +00efeb17bcaafce6,ANSSI-CC-2009/34,report,component_used,None +c175537338906951,ANSSI-CC-2016/70,report,component_shared,None +fb9ec9e846ea4e87,ANSSI-CC-2017/47,report,component_used,None +ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,report,component_used,None +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,report,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +cacdfdafce47678c,ANSSI-CC-2014/59,target,component_used,None +dba4b5a166f9456a,ANSSI-CC-2014/76,report,recertification,None +aa096ffde94b953b,BSI-DSZ-CC-0411-2007,report,component_used,None +b42009eb34746731,ANSSI-CC-2021/50,report,previous_version,maybe previous_version +463ecd64b7506048,ANSSI-CC-2018/52,report,component_used,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,report,recertification,None +c96343baf608174d,BSI-DSZ-CC-0645-2010,report,component_used,None +0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None +2c15dfc106ebf8d8,ANSSI-CC-2012/24,report,component_used,None +e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,unclear +ee1c6dd97918d74a,ANSSI-CC-2012/71,target,evaluation_reused,None +e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,report,component_used,None +845bb039719ac5d8,ANSSI-CC-2010/40,report,previous_version,None +af61a31e3fd0d6f0,ANSSI-CC-2010/02,report,evaluation_reused,None +d49988efd778ca9d,ANSSI-CC-2010/02,report,component_used,None +54273dd266fce692,ANSSI-CC-2015/15,report,component_used,None +4bede5d2af4cb11f,ANSSI-CC-2019/59,report,evaluation_reused,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,report,recertification,None +a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,report,component_used,None +f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,target,component_used,None +8bdb610131555c12,BSI-DSZ-CC-0645-2010,report,component_used,None +b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,unclear +371dca18821f7714,BSI-DSZ-CC-0945-2017,report,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,report,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,recertification,None +4f5f41ecf7517e63,ANSSI-CC-2016/80,report,evaluation_reused,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,report,recertification,None +16bdbde359584f99,ANSSI-CC-2017/61,report,component_used,None +34370b67b5e675c3,ANSSI-CC-2013/47,report,component_used,None +a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,report,previous_version,None +0bf7a19b22163465,ANSSI-CC-2015/80,report,evaluation_reused,None +1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,target,component_used,None +13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,report,component_shared,"BAC, EAC thingy" +f869956d14ea1694,ANSSI-CC-2012/68,report,component_used,None +037577fc2019fcfa,ANSSI-CC-2021/49,report,previous_version,None +17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,report,component_used,None +5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,target,component_used,None +1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,report,component_used,None +28e70f46ba4394dd,ANSSI-CC-2012/08,report,component_used,None +61832cb4291c343f,BSI-DSZ-CC-0955-2016,report,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0312-2005,report,component_used,None +627bfd69c2a831cf,ANSSI-CC-2014/37,report,evaluation_reused,None +aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,report,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 +05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,report,component_used,None +9529eb793550093c,ANSSI-CC-2018/40,report,previous_version,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,report,recertification,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,report,recertification,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,report,recertification,None +ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,target,component_used,None +30f71b100c5cebae,ANSSI-CC-2015/15,report,component_used,None +6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,target,component_used,None +06e505abb8dad1b8,ANSSI-CC-2017/54,report,component_used,None +317713948b4473b3,BSI-DSZ-CC-0798-2012,report,component_shared,None +055985699aee1e09,ANSSI-CC-2010/02,report,component_used,None +b61363e51ed90c7a,ANSSI-CC-2019/12,report,component_used,None +691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,report,component_used,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,report,recertification,None +19c60923e31777e7,ANSSI-CC-2017/49,report,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,report,component_used,None +c96343baf608174d,BSI-DSZ-CC-0750-2011,report,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,report,recertification,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,report,recertification,None +5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,target,component_used,unclear +4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component_used,None +c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,report,component_used,None +ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,report,component_used,None +30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,report,component_used,None +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,evaluation_reused,None +ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,report,component_used,None +bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,report,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" +97d00a9f198a6e57,ANSSI-CC-2015/66,report,component_used,None +744a7a202d909323,ANSSI-CC-2017/24,report,component_used,None +df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,report,component_used,None +6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,report,component_used,None +1b8705a0486b3be1,NSCIB-CC-0209053-CR,report,previous_version,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,report,recertification,None +cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,report,component_used,None +27d11629261d8806,ANSSI-CC-2014/59,report,component_used,None +a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,report,component_used,None +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,report,recertification,None +c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,"unclear, mentions just ToE delivery" +84a75b6fc33b669e,ANSSI-CC-2014/08,report,evaluation_reused,None +36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,report,component_used,None +01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,report,component_used,None +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,report,component_used,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv b/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv index 09368347..ffa897e7 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv @@ -1,101 +1,108 @@ dgst,referenced_cert_id,source,label,comment -a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,None,None,None -657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,None,None,None -0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,None,None,None -983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,None,None,None -8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,None,None,None -774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,None,None,None +a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,report,component_used,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,report,recertification,None +0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,report,component_used,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,report,recertification,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,report,recertification,None +774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,report,evaluation_reused,None c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,None,None,None -a5adb726852f5cc5,ANSSI-CC-2009/37,None,None,None -4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,None,None,None -cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,None,None,None -24de96ea505af909,ANSSI-CC-2021/29,None,None,None -2793414918738c7f,BSI-DSZ-CC-0857-2013,None,None,None -b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,None,None,None -d96438a7165e5d82,BSI-DSZ-CC-0410-2007,None,None,None -4b3f577896c80f8a,ANSSI-CC-2015/80,None,None,None -22388445fe620ac0,ANSSI-CC-2014/20,None,None,None -2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,None,None,None -6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,None,None,None -120a49c0aa284f68,BSI-DSZ-CC-0813-2012,None,None,None -994ac7d986a99e50,BSI-DSZ-CC-0640-2010,None,None,None -16abf8ee0697e64f,ANSSI-CC-2014/61,None,None,None -f05de4db911ec8af,CRP243,None,None,None -38f79f1a48e7844f,ANSSI-CC-2017/47,None,None,None -fe55e99f4f3dc195,ANSSI-CC-2013/33,None,None,None -fc875e0381a28435,BSI-DSZ-CC-0432-2007,None,None,None -c23bb542201aa511,BSI-DSZ-CC-0829-2012,None,None,None -2532fe8c7214c485,BSI-DSZ-CC-0675-2011,None,None,None -2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,None,None,None -833ff87d69cff5db,383-4-184,None,None,None -c7c1215819127a1c,BSI-DSZ-CC-0322-2005,None,None,None -647d17d44745a532,BSI-DSZ-CC-0904-2015,None,None,None -4489bfc781a82281,BSI-DSZ-CC-0817-2013,None,None,None -94fb86d31446e531,BSI-DSZ-CC-0958-2015,None,None,None -37bae9eaf342542b,BSI-DSZ-CC-0349-2006,None,None,None -5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,None,None,None -5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,None,None,None -d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,None,None,None -0f53257eb74bee54,BSI-DSZ-CC-0417-2008,None,None,None -84d6a370e4f38ddc,ANSSI-CC-2019/12,None,None,None -1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,None,None,None -d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,None,None,None -38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,None,None,None -e6057ffc5085a192,ANSSI-CC-2014/48,None,None,None -db7627e56b8621b1,BSI-DSZ-CC-0639-2010,None,None,None -4b3f577896c80f8a,ANSSI-CC-2016/44,None,None,None -5f3c9c6bf76ba72d,ANSSI-CC-2018/51,None,None,None -22d6578b30f5a227,BSI-DSZ-CC-1107-2020,None,None,None -a0c38b4389ad7cf7,ANSSI-CC-2018/12,None,None,None -19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,None,None,None -f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,None,None -2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,None,None,None -3357c28c91cc2ed1,ANSSI-CC-2016/44,None,None,None -f6317f134f532696,ANSSI-CC-2017/24,None,None,None -234da54efbcf734b,BSI-DSZ-CC-0917-2014,None,None,None -1b20b1629ba7d3a6,ANSSI-CC-2012/72,None,None,None -2532fe8c7214c485,BSI-DSZ-CC-0633-2010,None,None,None -b1ef468658a9a0c1,NSCIB-CC-0448219-CR,None,None,None -35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,None,None,None -edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,None,None,None -3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,None,None,None -0fe8df0e85116b61,ANSSI-CC-2010/33,None,None,None -bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,None,None,None -b00bb452e7a8176c,ANSSI-CC-2016/57,None,None,None -7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,None,None,None -81273108dd167b98,BSI-DSZ-CC-0523-2008,None,None,None -63c1045055ec3b58,BSI-DSZ-CC-0266-2005,None,None,None -a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,None,None,None -4c017087976d05eb,ANSSI-CC-2020/45,None,None,None -f4d510a39278e687,ANSSI-CC-2018/32,None,None,None -c23bb542201aa511,ANSSI-CC-2014/14,None,None,None -625b243397001f01,BSI-DSZ-CC-1059-2018,None,None,None -cae968d4926648bd,BSI-DSZ-CC-0754-2012,None,None,None -000176284683faa2,BSI-DSZ-CC-0640-2010,None,None,None -66faa094a9333d1a,ANSSI-CC-2014/86,None,None,None -a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,None,None,None -031667f4e242da61,CCEVS-VR-07-0054,None,None,None -459539da9315ccac,ANSSI-CC-2021/30,None,None,None -69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,None,None,None -a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,None,None,None -50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,None,None,None -5cf51622fafaf868,BSI-DSZ-CC-1005-2016,None,None,None -1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,None,None,None -0c7ef6c32cbdee47,ANSSI-CC-2017/61,None,None,None -8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,None,None,None -6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,None,None,None -4962c0d8106b8b5e,ANSSI-CC-2015/37,None,None,None -e6057ffc5085a192,BSI-DSZ-CC-0829-2012,None,None,None -03fad65e5af65088,BSI-DSZ-CC-0696-2011,None,None,None -720aab4f782f4c60,BSI-DSZ-CC-0555-2009,None,None,None -9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,None,None,None -d55eb8c97d71e843,ANSSI-CC-2021/49,None,None,None -73e0b884cd04d9bc,NSCIB-CC-12-36243,None,None,None -2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,None,None,None -f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,None,None,None -f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,None,None,None -2f43fe24bd8e91e1,ANSSI-CC-2020/65,None,None,None -d96438a7165e5d82,BSI-DSZ-CC-0411-2007,None,None,None -7623950728f52a71,ANSSI-CC-2016/06,None,None,None -ba69597434946260,BSI-DSZ-CC-0293-2005,None,None,None -50aef770cb4f028f,BSI-DSZ-CC-1059-2018,None,None,None +a5adb726852f5cc5,ANSSI-CC-2009/37,report,component_used,None +4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,report,component_used,None +cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,report,component_used,None +24de96ea505af909,ANSSI-CC-2021/29,report,component_used,None +2793414918738c7f,BSI-DSZ-CC-0857-2013,target,component_used,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,report,recertification,None +d96438a7165e5d82,BSI-DSZ-CC-0410-2007,report,component_used,None +4b3f577896c80f8a,ANSSI-CC-2015/80,report,evaluation_reused,None +22388445fe620ac0,ANSSI-CC-2014/20,report,evaluation_reused,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,report,recertification,None +6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,report,component_used,None +120a49c0aa284f68,BSI-DSZ-CC-0813-2012,report,component_used,None +994ac7d986a99e50,BSI-DSZ-CC-0640-2010,report,component_used,None +16abf8ee0697e64f,ANSSI-CC-2014/61,report,component_shared,None +f05de4db911ec8af,CRP243,report,previous_version,None +38f79f1a48e7844f,ANSSI-CC-2017/47,report,component_used,None +fe55e99f4f3dc195,ANSSI-CC-2013/33,report,evaluation_reused,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,report,recertification,None +c23bb542201aa511,BSI-DSZ-CC-0829-2012,report,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0675-2011,report,component_shared,None +2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,report,component_used,None +833ff87d69cff5db,383-4-184,None,None,unclear +c7c1215819127a1c,BSI-DSZ-CC-0322-2005,report,component_used,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,report,recertification,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,report,recertification,None +94fb86d31446e531,BSI-DSZ-CC-0958-2015,report,component_used,None +37bae9eaf342542b,BSI-DSZ-CC-0349-2006,report,evaluation_reused,None +5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,report,component_used,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,report,recertification,None +d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,report,component_used,None +0f53257eb74bee54,BSI-DSZ-CC-0417-2008,report,component_used,unclear +84d6a370e4f38ddc,ANSSI-CC-2019/12,report,component_used,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,report,recertification,None +d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,report,component_used,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,report,recertification,None +e6057ffc5085a192,ANSSI-CC-2014/48,report,evaluation_reused,None +db7627e56b8621b1,BSI-DSZ-CC-0639-2010,report,component_used,None +4b3f577896c80f8a,ANSSI-CC-2016/44,report,evaluation_reused,None +5f3c9c6bf76ba72d,ANSSI-CC-2018/51,report,component_used,None +22d6578b30f5a227,BSI-DSZ-CC-1107-2020,report,component_used,None +a0c38b4389ad7cf7,ANSSI-CC-2018/12,report,component_used,None +19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,report,component_used,None +f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,None,unclear +2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,report,component_used,None +3357c28c91cc2ed1,ANSSI-CC-2016/44,report,evaluation_reused,None +f6317f134f532696,ANSSI-CC-2017/24,report,previous_version,None +234da54efbcf734b,BSI-DSZ-CC-0917-2014,report,component_used,None +1b20b1629ba7d3a6,ANSSI-CC-2012/72,target,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0633-2010,report,component_used,None +b1ef468658a9a0c1,NSCIB-CC-0448219-CR,report,evaluation_reused,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,report,recertification,None +edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,report,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. +Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of +technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the +TOE was re-evaluated because of changes on top layers for modified module +packaging requirements. Additional package types (MOB4 and complete +passport inlay) and the relevant production sites were included in the evaluation +process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the +TOE itself. The Security Target was updated. " +3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,report,component_used,None +0fe8df0e85116b61,ANSSI-CC-2010/33,report,previous_version,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,report,recertification,None +b00bb452e7a8176c,ANSSI-CC-2016/57,report,previous_version,None +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,report,recertification,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,report,recertification,None +63c1045055ec3b58,BSI-DSZ-CC-0266-2005,report,component_used,None +a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,report,component_used,None +4c017087976d05eb,ANSSI-CC-2020/45,report,evaluation_reused,None +f4d510a39278e687,ANSSI-CC-2018/32,report,component_used,None +c23bb542201aa511,ANSSI-CC-2014/14,report,previous_version,None +625b243397001f01,BSI-DSZ-CC-1059-2018,target,component_used,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,report,recertification,None +000176284683faa2,BSI-DSZ-CC-0640-2010,report,component_used,None +66faa094a9333d1a,ANSSI-CC-2014/86,report,evaluation_reused,None +a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,report,component_used,None +031667f4e242da61,CCEVS-VR-07-0054,report,component_used,None +459539da9315ccac,ANSSI-CC-2021/30,report,previous_version,None +69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,report,component_used,None +a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,report,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,report,component_used,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,report,recertification,None +1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,report,component_used,None +0c7ef6c32cbdee47,ANSSI-CC-2017/61,report,component_used,None +8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,report,component_used,None +6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,report,component_used,None +4962c0d8106b8b5e,ANSSI-CC-2015/37,report,evaluation_reused,None +e6057ffc5085a192,BSI-DSZ-CC-0829-2012,report,component_used,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,report,recertification,None +720aab4f782f4c60,BSI-DSZ-CC-0555-2009,report,component_used,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,report,recertification,None +d55eb8c97d71e843,ANSSI-CC-2021/49,report,previous_version,None +73e0b884cd04d9bc,NSCIB-CC-12-36243,report,component_used,None +2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,report,component_used,None +f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,report,component_used,None +f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None +2f43fe24bd8e91e1,ANSSI-CC-2020/65,report,component_used,None +d96438a7165e5d82,BSI-DSZ-CC-0411-2007,report,component_used,None +7623950728f52a71,ANSSI-CC-2016/06,report,component_used,None +ba69597434946260,BSI-DSZ-CC-0293-2005,report,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-2018,report,component_used,None -- cgit v1.3.1 From 2fdc0c2f3420adb6a0ef0f562b103966d10a07a4 Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Fri, 8 Sep 2023 14:17:49 +0200 Subject: enrich train dataset with some samples from test --- .../manual_annotations/train.csv | 298 +++-- .../data/reference_annotations/split/test.json | 1386 ++++++++++---------- .../data/reference_annotations/split/train.json | 920 +++++++------ 3 files changed, 1350 insertions(+), 1254 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv index ca8f2774..3d5a0ab5 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv @@ -1,101 +1,197 @@ -dgst,referenced_cert_id,source,label,comment -e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,report,component_used,the previously certified M7892 G12 component -0bf7a19b22163465,ANSSI-CC-2016/44,report,evaluation_reused,unclear whether component used -ab3af998dff7a2ef,ANSSI-CC-2017/47,report,component_used,None -8ba22f6c9651edc3,ANSSI-CC-2013/55,report,component_used,None -680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,report,component_used,None -3515801dee00995f,BSI-DSZ-CC-0447-2008,report,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 -6a999675c9422dfb,BSI-DSZ-CC-0353-2006,report,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 -12e20466a0f08342,ANSSI-CC-2016/79,report,evaluation_reused,unclear whether component used -d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,report,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert -921e042759b30033,BSI-DSZ-CC-0978-2016,report,component_used,composition scheme “[COMP]” mentioned -00efeb17bcaafce6,ANSSI-CC-2009/34,report,component_used,None -c175537338906951,ANSSI-CC-2016/70,report,component_shared,None -fb9ec9e846ea4e87,ANSSI-CC-2017/47,report,component_used,None -ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,report,component_used,None -f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,report,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 -cacdfdafce47678c,ANSSI-CC-2014/59,target,component_used,None -dba4b5a166f9456a,ANSSI-CC-2014/76,report,recertification,None -aa096ffde94b953b,BSI-DSZ-CC-0411-2007,report,component_used,None -b42009eb34746731,ANSSI-CC-2021/50,report,previous_version,maybe previous_version -463ecd64b7506048,ANSSI-CC-2018/52,report,component_used,None -c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,report,recertification,None -c96343baf608174d,BSI-DSZ-CC-0645-2010,report,component_used,None -0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None -2c15dfc106ebf8d8,ANSSI-CC-2012/24,report,component_used,None -e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,unclear -ee1c6dd97918d74a,ANSSI-CC-2012/71,target,evaluation_reused,None -e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,report,component_used,None -845bb039719ac5d8,ANSSI-CC-2010/40,report,previous_version,None -af61a31e3fd0d6f0,ANSSI-CC-2010/02,report,evaluation_reused,None -d49988efd778ca9d,ANSSI-CC-2010/02,report,component_used,None -54273dd266fce692,ANSSI-CC-2015/15,report,component_used,None -4bede5d2af4cb11f,ANSSI-CC-2019/59,report,evaluation_reused,None -d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,report,recertification,None -a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,report,component_used,None -f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,target,component_used,None -8bdb610131555c12,BSI-DSZ-CC-0645-2010,report,component_used,None -b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,unclear -371dca18821f7714,BSI-DSZ-CC-0945-2017,report,component_used,None -f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,report,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 -e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,recertification,None -4f5f41ecf7517e63,ANSSI-CC-2016/80,report,evaluation_reused,None -a867d281d34d34b7,BSI-DSZ-CC-0976-2015,report,recertification,None -16bdbde359584f99,ANSSI-CC-2017/61,report,component_used,None -34370b67b5e675c3,ANSSI-CC-2013/47,report,component_used,None -a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,report,previous_version,None -0bf7a19b22163465,ANSSI-CC-2015/80,report,evaluation_reused,None -1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,target,component_used,None -13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,report,component_shared,"BAC, EAC thingy" -f869956d14ea1694,ANSSI-CC-2012/68,report,component_used,None -037577fc2019fcfa,ANSSI-CC-2021/49,report,previous_version,None -17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,report,component_used,None -5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,target,component_used,None -1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,report,component_used,None -28e70f46ba4394dd,ANSSI-CC-2012/08,report,component_used,None -61832cb4291c343f,BSI-DSZ-CC-0955-2016,report,component_used,None -42799014c183e1b3,BSI-DSZ-CC-0312-2005,report,component_used,None -627bfd69c2a831cf,ANSSI-CC-2014/37,report,evaluation_reused,None -aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,report,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 -05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,report,component_used,None -9529eb793550093c,ANSSI-CC-2018/40,report,previous_version,None -7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,report,recertification,None -ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,report,recertification,None -ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,report,recertification,None -ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,target,component_used,None -30f71b100c5cebae,ANSSI-CC-2015/15,report,component_used,None -6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,target,component_used,None -06e505abb8dad1b8,ANSSI-CC-2017/54,report,component_used,None -317713948b4473b3,BSI-DSZ-CC-0798-2012,report,component_shared,None -055985699aee1e09,ANSSI-CC-2010/02,report,component_used,None -b61363e51ed90c7a,ANSSI-CC-2019/12,report,component_used,None -691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,report,component_used,None -a07548a64fac5794,BSI-DSZ-CC-0350-2007,report,recertification,None -19c60923e31777e7,ANSSI-CC-2017/49,report,component_used,None -f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,report,component_used,None -c96343baf608174d,BSI-DSZ-CC-0750-2011,report,component_used,None -42799014c183e1b3,BSI-DSZ-CC-0429-2007,report,recertification,None -3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,report,recertification,None -5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,target,component_used,unclear -4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component_used,None -c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,report,component_used,None -ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,report,component_used,None -30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,report,component_used,None -c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,evaluation_reused,None -ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,report,component_used,None -bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,report,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" -97d00a9f198a6e57,ANSSI-CC-2015/66,report,component_used,None -744a7a202d909323,ANSSI-CC-2017/24,report,component_used,None -df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,report,component_used,None -6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,report,component_used,None -1b8705a0486b3be1,NSCIB-CC-0209053-CR,report,previous_version,None -a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,report,recertification,None -cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,report,component_used,None -27d11629261d8806,ANSSI-CC-2014/59,report,component_used,None -a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,report,component_used,None -d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,report,recertification,None -c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,"unclear, mentions just ToE delivery" -84a75b6fc33b669e,ANSSI-CC-2014/08,report,evaluation_reused,None -36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,report,component_used,None -01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,report,component_used,None -0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,report,component_used,None +,dgst,referenced_cert_id,source,label,comment +45,00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None +38,00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None +39,00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None +87,00b539d629e60965,CCEVS-VR-02-0025,None,None,None +79,00eb3e5c7a4537b1,BSI-DSZ-CC-0399-2007,None,None,None +80,00eb3e5c7a4537b1,BSI-DSZ-CC-0322-2005,None,None,None +10,00efeb17bcaafce6,ANSSI-CC-2009/34,report,component_used,None +37,017f3cea8275a3e1,ANSSI-CC-2017/23,None,None,None +2,01ce6f422bec823d,BSI-DSZ-CC-1040-2019,None,None,None +26,01dc77d7f8f94d97,ANSSI-CC-2017/39,None,None,None +27,01dc77d7f8f94d97,ANSSI-CC-2016/78,None,None,None +49,01e5c6d31f763f56,BSI-DSZ-CC-0917-2014,None,None,None +98,01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,report,component_used,None +46,0208e6b60a0b1a32,ANSSI-CC-2016/01,None,None,None +0,02c6fbcd90dcf05a,BSI-DSZ-CC-1107-V3-2022,None,None,None +20,02cb3c385865ad2c,BSI-DSZ-CC-1184-2022,None,None,None +21,0319b03323f257d1,BSI-DSZ-CC-1121-2019,None,None,None +22,0319b03323f257d1,BSI-DSZ-CC-1118-2020,None,None,None +47,032faef0207b715f,BSI-DSZ-CC-0837-V2-2014,None,None,None +49,037577fc2019fcfa,ANSSI-CC-2021/49,report,previous_version,None +48,04c5a5a66fb7f89a,BSI-DSZ-CC-0827-V2-2014,None,None,None +78,04cb360502da6492,BSI-DSZ-CC-0344-2005,None,None,None +95,054bd8196e015197,ANSSI-CC-2011/07,None,None,None +68,055985699aee1e09,ANSSI-CC-2010/02,report,component_used,None +58,05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,report,component_used,None +85,05d96aada7bb839c,BSI-DSZ-CC-0223-2003,None,None,None +60,06b24bc51eb69188,BSI-DSZ-CC-0813-2012,None,None,None +66,06e505abb8dad1b8,ANSSI-CC-2017/54,report,component_used,None +44,06e70fa70393d60f,BSI-DSZ-CC-0891-V2-2016,None,None,None +43,06e70fa70393d60f,ANSSI-CC-2017/07,None,None,None +33,075ddd00e07fa7ed,ANSSI-CC-2020/76,None,None,None +34,075ddd00e07fa7ed,BSI-DSZ-CC-1136-2021,None,None,None +24,07ce7e4844f62c5c,BSI-DSZ-CC-1070-2018,None,None,None +25,07ce7e4844f62c5c,ANSSI-CC-2017/73,None,None,None +64,080aa6966d757f05,ANSSI-CC-2010/02,None,None,None +63,080aa6966d757f05,ANSSI-CC-2010/05,None,None,None +17,08408cbc394e7421,BSI-DSZ-CC-0782-V4-2018,None,None,None +18,08408cbc394e7421,ANSSI-CC-2017/59,None,None,None +13,09220515a7edf3f9,BSI-DSZ-CC-1110-V2-2019,None,None,None +77,096b05114ee5e09f,BSI-DSZ-CC-0411-2007,None,None,None +76,096b05114ee5e09f,BSI-DSZ-CC-0410-2007,None,None,None +75,096b05114ee5e09f,ANSSI-CC-2008/45,None,None,None +90,096ccb6bcb3d2731,BSI-DSZ-CC-0564-2009,None,None,None +19,09b07fbbb24e1b04,BSI-DSZ-CC-0964-2015,None,None,None +72,09f539aedefa9756,ANSSI-CC-2008/05,None,None,None +70,09f539aedefa9756,ANSSI-CC-2009/11,None,None,None +71,09f539aedefa9756,ANSSI-CC-2009/41,None,None,None +40,0a6e50445a906052,BSI-DSZ-CC-0945-2017,None,None,None +41,0a6e50445a906052,BSI-DSZ-CC-0891-V2-2016,None,None,None +66,0aad0176e35b3749,BSI-DSZ-CC-0477-2007,None,None,None +22,0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None +31,0bd651fcbb86f2a9,BSI-DSZ-CC-1136-2021,None,None,None +32,0bd651fcbb86f2a9,ANSSI-CC-2020/74,None,None,None +45,0bf7a19b22163465,ANSSI-CC-2015/80,report,evaluation_reused,None +1,0bf7a19b22163465,ANSSI-CC-2016/44,report,evaluation_reused,unclear whether component used +58,0c9e215997d96f44,BSI-DSZ-CC-0555-2009,None,None,None +88,0cd406e7c5013b74,BSI-DSZ-CC-0304-2006,None,None,None +53,0d4a95a8b52b787b,BSI-DSZ-CC-0829-2012,None,None,None +30,0da9d81c9f563df5,BSI-DSZ-CC-0382-2007,None,None,None +62,0ee132dec087b772,ANSSI-CC-2011/07,None,None,None +1,0ef56cba2362e463,ANSSI-CC-2015/42,None,None,None +59,0f1b31ac93459d1f,BSI-DSZ-CC-0720-2011,None,None,None +99,0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,report,component_used,None +65,0f684159ad31f883,BSI-DSZ-CC-0555-2009,None,None,None +86,0fe0bb1054f49b7f,CCEVS-VR-04-0092,None,None,None +89,0ff82b1cc24b3105,BSI-DSZ-CC-0302-2005,None,None,None +11,10233188161519ee,ANSSI-CC-2020/26,None,None,None +10,10233188161519ee,BSI-DSZ-CC-1059-V3-2019,None,None,None +12,10233188161519ee,BSI-DSZ-CC-1059-2018,None,None,None +9,10233188161519ee,ANSSI-CC-2019/48,None,None,None +8,102707a8c4d8a1b0,BSI-DSZ-CC-1059-2018,None,None,None +7,102707a8c4d8a1b0,ANSSI-CC-2020/26,None,None,None +6,102707a8c4d8a1b0,ANSSI-CC-2019/35,None,None,None +5,102707a8c4d8a1b0,BSI-DSZ-CC-1059-V3-2019,None,None,None +15,102bbdfcd696d7a9,ANSSI-CC-2019/01,None,None,None +16,102bbdfcd696d7a9,ANSSI-CC-2018/13,None,None,None +23,107c348f4ba0a41c,BSI-DSZ-CC-1135-2020,None,None,None +67,109eb2158ca6a2f9,BSI-DSZ-CC-0410-2007,None,None,None +52,10af4a6378d0fb7a,ANSSI-CC-2014/07,None,None,None +50,10af4a6378d0fb7a,BSI-DSZ-CC-0837-V2-2014,None,None,None +51,10af4a6378d0fb7a,ANSSI-CC-2014/86,None,None,None +35,10b17081dd7cad8f,ANSSI-CC-2017/39,None,None,None +36,10b17081dd7cad8f,ANSSI-CC-2015/36,None,None,None +69,112fe9f123e277ee,ANSSI-CC-2010/02,None,None,None +68,112fe9f123e277ee,ANSSI-CC-2010/19,None,None,None +3,114e39b4da949e6d,BSI-DSZ-CC-1110-V2-2019,None,None,None +4,114e39b4da949e6d,ANSSI-CC-2020/08,None,None,None +29,1172a88e009dc248,CRP281,None,None,None +28,1172a88e009dc248,CRP257,None,None,None +73,1188aebd8681e4fe,BSI-DSZ-CC-0465-2008,None,None,None +82,122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None,None +81,122245dd15683e26,ANSSI-CC-2007/08,None,None,None +91,123c94418ddf1ad8,BSI-DSZ-CC-0367-2006,None,None,None +7,12e20466a0f08342,ANSSI-CC-2016/79,report,evaluation_reused,unclear whether component used +14,131ec924c42ef90b,ANSSI-CC-2017/61,None,None,None +42,135ca3602f0005bf,NSCIB-CC-16-99111-CR2,None,None,None +47,13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,report,component_shared,"BAC, EAC thingy" +83,13e5b88ad3becb77,BSI-DSZ-CC-0322-2005,None,None,None +84,13e5b88ad3becb77,BSI-DSZ-CC-0344-2005,None,None,None +61,1468948b4f0509ab,BSI-DSZ-CC-0813-2012,None,None,None +74,157b741a0f529a0a,ANSSI-CC-2009/37,None,None,None +56,157f811757a0b29e,BSI-DSZ-CC-0675-2011,None,None,None +57,157f811757a0b29e,BSI-DSZ-CC-0633-2010,None,None,None +55,157f811757a0b29e,BSI-DSZ-CC-0555-2009,None,None,None +54,157f811757a0b29e,BSI-DSZ-CC-0745-2012,None,None,None +92,158950ed4bc35274,ANSSI-CC-2016/33,None,None,None +93,158950ed4bc35274,ANSSI-CC-2015/80,None,None,None +94,158950ed4bc35274,ANSSI-CC-2015/36,None,None,None +42,16bdbde359584f99,ANSSI-CC-2017/61,report,component_used,None +50,17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,report,component_used,None +72,19c60923e31777e7,ANSSI-CC-2017/49,report,component_used,None +89,1b8705a0486b3be1,NSCIB-CC-0209053-CR,report,previous_version,None +46,1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,target,component_used,None +52,1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,report,component_used,None +92,27d11629261d8806,ANSSI-CC-2014/59,report,component_used,None +53,28e70f46ba4394dd,ANSSI-CC-2012/08,report,component_used,None +23,2c15dfc106ebf8d8,ANSSI-CC-2012/24,report,component_used,None +81,30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,report,component_used,None +64,30f71b100c5cebae,ANSSI-CC-2015/15,report,component_used,None +67,317713948b4473b3,BSI-DSZ-CC-0798-2012,report,component_shared,None +43,34370b67b5e675c3,ANSSI-CC-2013/47,report,component_used,None +5,3515801dee00995f,BSI-DSZ-CC-0447-2008,report,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 +97,36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,report,component_used,None +37,371dca18821f7714,BSI-DSZ-CC-0945-2017,report,component_used,None +76,3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,report,recertification,None +75,42799014c183e1b3,BSI-DSZ-CC-0429-2007,report,recertification,None +55,42799014c183e1b3,BSI-DSZ-CC-0312-2005,report,component_used,None +19,463ecd64b7506048,ANSSI-CC-2018/52,report,component_used,None +78,4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component_used,None +31,4bede5d2af4cb11f,ANSSI-CC-2019/59,report,evaluation_reused,None +40,4f5f41ecf7517e63,ANSSI-CC-2016/80,report,evaluation_reused,None +30,54273dd266fce692,ANSSI-CC-2015/15,report,component_used,None +51,5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,target,component_used,None +77,5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,target,component_used,unclear +54,61832cb4291c343f,BSI-DSZ-CC-0955-2016,report,component_used,None +56,627bfd69c2a831cf,ANSSI-CC-2014/37,report,evaluation_reused,None +4,680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,report,component_used,None +70,691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,report,component_used,None +6,6a999675c9422dfb,BSI-DSZ-CC-0353-2006,report,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +65,6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,target,component_used,None +88,6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,report,component_used,None +86,744a7a202d909323,ANSSI-CC-2017/24,report,component_used,None +60,7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,report,recertification,None +27,845bb039719ac5d8,ANSSI-CC-2010/40,report,previous_version,None +96,84a75b6fc33b669e,ANSSI-CC-2014/08,report,evaluation_reused,None +3,8ba22f6c9651edc3,ANSSI-CC-2013/55,report,component_used,None +35,8bdb610131555c12,BSI-DSZ-CC-0645-2010,report,component_used,None +9,921e042759b30033,BSI-DSZ-CC-0978-2016,report,component_used,composition scheme “[COMP]” mentioned +59,9529eb793550093c,ANSSI-CC-2018/40,report,previous_version,None +85,97d00a9f198a6e57,ANSSI-CC-2015/66,report,component_used,None +71,a07548a64fac5794,BSI-DSZ-CC-0350-2007,report,recertification,None +33,a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,report,component_used,None +44,a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,report,previous_version,None +93,a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,report,component_used,None +41,a867d281d34d34b7,BSI-DSZ-CC-0976-2015,report,recertification,None +90,a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,report,recertification,None +17,aa096ffde94b953b,BSI-DSZ-CC-0411-2007,report,component_used,None +80,ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,report,component_used,None +2,ab3af998dff7a2ef,ANSSI-CC-2017/47,report,component_used,None +57,aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,report,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 +28,af61a31e3fd0d6f0,ANSSI-CC-2010/02,report,evaluation_reused,None +36,b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,unclear +18,b42009eb34746731,ANSSI-CC-2021/50,report,previous_version,maybe previous_version +69,b61363e51ed90c7a,ANSSI-CC-2019/12,report,component_used,None +84,bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,report,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" +79,c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,report,component_used,None +11,c175537338906951,ANSSI-CC-2016/70,report,component_shared,None +82,c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,evaluation_reused,None +95,c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,"unclear, mentions just ToE delivery" +20,c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,report,recertification,None +21,c96343baf608174d,BSI-DSZ-CC-0645-2010,report,component_used,None +74,c96343baf608174d,BSI-DSZ-CC-0750-2011,report,component_used,None +15,cacdfdafce47678c,ANSSI-CC-2014/59,target,component_used,None +91,cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,report,component_used,None +8,d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,report,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert +94,d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,report,recertification,None +29,d49988efd778ca9d,ANSSI-CC-2010/02,report,component_used,None +32,d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,report,recertification,None +16,dba4b5a166f9456a,ANSSI-CC-2014/76,report,recertification,None +87,df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,report,component_used,None +24,e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,unclear +39,e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,recertification,None +0,e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,report,component_used,the previously certified M7892 G12 component +26,e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,report,component_used,None +62,ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,report,recertification,None +63,ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,target,component_used,None +13,ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,report,component_used,None +83,ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,report,component_used,None +61,ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,report,recertification,None +25,ee1c6dd97918d74a,ANSSI-CC-2012/71,target,evaluation_reused,None +73,f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,report,component_used,None +38,f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,report,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 +34,f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,target,component_used,None +14,f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,report,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +48,f869956d14ea1694,ANSSI-CC-2012/68,report,component_used,None +12,fb9ec9e846ea4e87,ANSSI-CC-2017/47,report,component_used,None diff --git a/src/sec_certs/data/reference_annotations/split/test.json b/src/sec_certs/data/reference_annotations/split/test.json index 41ac06ee..2e761490 100644 --- a/src/sec_certs/data/reference_annotations/split/test.json +++ b/src/sec_certs/data/reference_annotations/split/test.json @@ -1,780 +1,714 @@ [ - "11a98c5d5ff6f40f", - "6dde54b8c6d592ef", - "054bd8196e015197", - "c902a788d699fe4a", - "5d674c06a559a5e7", - "9f8c9a3aa64a9f8e", - "9545f2b0b8b92c8d", - "56ceb1a9c003ac3f", - "9299f6cd8a266541", - "d1fe94ee32cbbf05", - "6313a92bf426d2bb", - "e8a37e48a8d34410", - "f34df0dd7011366e", - "c23218b044706800", - "3f4b6e4f245f6fab", - "5f99b00bf258f32f", - "4f3bf0c433b7f54f", - "587055293e9fa51d", - "da2d3567a814d2d7", - "915e919390e6f58b", - "a5e8267322d0d7a6", - "82dce1546c69369d", - "7d4585a4b5b6e873", - "736ede0289146b59", + "0160f6becf137107", + "05822f92f304fcd6", + "06998c36415036aa", + "08bff656ff61e949", + "0a6525cdbeb38df6", "0cbcee9f1cde47a8", - "4bf7d6ef245dd393", - "3817c0aca007989e", - "baead73ecc6e4a33", + "0f7ce013d15f6ff5", + "10229cdd92c1989f", + "13d2556ef392ed41", + "15ae64b85b3e28d7", + "15b121492722bffb", + "15ee7651b7ad610c", + "15fae84c1e6c6c57", + "16513a1bff79b46a", + "16b0a0811bc4fe82", + "16b24ff1bf3c079b", + "16f1a518cd842fcb", + "173097995b7a7f12", + "173704f0d2b8a02f", + "177dee4c1051e612", + "17bfc3d0570ab3d0", + "183a5cd3e2aaa0ec", + "18a515a0fff0c0c3", + "18b7e5bdf459ca13", + "195366bfba7213a5", + "1999d4ed82b3188e", "19d3185c40be008d", - "8a7731e9eb363fb4", - "78a279b07a6d3367", - "47913a485c3c8a18", - "5f259ce91f0234a0", - "d4a14ea7adb375ed", - "994a2bf994b4facd", - "ac1257069b2f4afd", - "b5985efd378af972", - "bc1452fff95744b9", - "4f953ae00a24eb85", - "a35ed74b7ea3ec4b", - "654554952ecb20ac", - "157b741a0f529a0a", - "30ae700d801bd41c", - "2f9feaf4121720da", - "e0283686d26420d9", + "1ac8de53c0894d18", "1b2764a62ffe86a2", - "b47b457f73c42c60", - "ff45605151ec4eb1", - "5346d5a4418466ab", - "e5b04a0a3872954f", - "981f7ea91d7bdcb6", - "f229a8078cb4ff82", - "731aac44217611ec", - "4e29916c366fe997", - "5efe98a1ba4df4d7", - "ffd213e55eee8cc8", - "7e24168630d5124e", - "00ad121fa63e5dfb", - "173097995b7a7f12", - "4b2f963b48e0f954", - "5e4c757231135ba1", - "aed7a612d966a786", - "1468948b4f0509ab", - "4d885e54f1361d04", - "8a5f8686ab9de19e", - "37d28734245a3bb1", - "80fe3ba3d9c9bf64", - "b1b385d5bad724ed", - "c4ca6565698f0e49", - "d9bfffa3cc6d1c53", - "d32e5d4d5ff8b871", - "8865aed38c79d3fc", - "5bf118ffa29520c0", - "6136acacc653106c", - "05d96aada7bb839c", - "0aad0176e35b3749", - "64ed7e55bdc50177", - "3976c9e492193315", - "65fba51313f65938", - "55e52bcaab69d573", - "4dd0daec78093d31", - "d67edade78d25033", - "71eae9be60b7fb6c", - "3d149fe7c08bfc58", + "1bcfaefe46abccf0", + "1c8567a1b1a6c12a", + "1cc05dbb992431b9", + "1d1df0fb541e49b8", "1d9ae732c5dec242", - "6bdca9f13e19572e", - "d6ade0a63d0f9130", - "da1139ecdc3512da", - "637657613004dbbd", + "1ea31bb5f6a15995", + "1ff08f79cb89c1de", + "2078424be58e4db1", + "21951e191e55b66e", + "21cffadbbb87c205", + "22ede463dbf1a105", + "23a85a7f9c07412f", + "24624a4e60ddedd4", + "259df98e7476a843", + "259e0ab1c4b690cf", + "25c07c5d0a05d86c", + "26c4912b140fde9f", + "26c7b173668167d8", + "2760aeedce0b79db", + "27ccec0740bb7915", + "296618f10b019d9c", + "29cc495628e76ffa", + "29f2dd90f57ea948", "29fb07d8f74e734b", - "075ddd00e07fa7ed", - "ce92195cf4a19084", - "68875c05e00bd1aa", - "61e862ed07d3cb03", - "cdbbbad526f0aa13", - "1d1df0fb541e49b8", - "5d4aae047020163b", - "c67724216ed5a318", - "10229cdd92c1989f", - "4dbb108406acb1a5", - "9292ab2134e8a712", - "84501b4439b66fd9", + "2a07b85f61ac08b0", + "2a91f42389fda90d", + "2adb03f5294c31c2", + "2b0342d5b70b7a78", + "2bd9661657e78278", + "2be10f342e68a89e", + "2c41f9dfcd5014e6", + "2d046178aa118fff", + "2d0b4100c3ead88a", + "2e997ba7beaed34d", + "2eee0fdd5cdaf565", + "2f9feaf4121720da", + "30ae700d801bd41c", + "30c6777901ae346c", + "30e9ce0969941ffb", + "3183669bf78db43b", + "31fd07069aebf013", + "328fee1e52f7ac82", + "3334d22b6559d0e2", + "337ece90615ed69d", + "3392558f04e04663", + "33cada8c95bf55c5", + "34506b8fab6dd7c3", + "34ed76f0c32d9e93", + "3519c14e4114d93d", + "35b9be2203d2b9e6", "35efe13fa9e93a68", - "5c5431cca026fb3e", - "e3a743ef04ecfb64", - "89d1f7ab8b77e41d", - "70908a4cfdc5585c", - "8035f242b12cec6d", - "67240adc18be338d", + "36641ce7192b5f92", + "36df04ab9c978ab9", + "373e7b1bc9066563", + "3746512864b941d4", + "37b94c08f8e25249", + "37c8d23c44b95833", + "37d28734245a3bb1", + "37e0d3cb098458d0", + "37fe6036c2ac932b", + "3817c0aca007989e", + "384dca297bd4f1ca", + "3976c9e492193315", + "39c07ebbca541145", + "3a3b0a9113835307", + "3a6a5f536bfbd4d4", + "3ba9f330ce93636b", + "3d149fe7c08bfc58", + "3d756c83419bba28", + "3d8083b1e6c7b336", "3e08a27e9d9c9b1e", - "5eb91b620bbf5bfc", - "7e92dcac20e009a5", + "3e182053b03f1faf", + "3e345d3c6002ad55", + "3ea02a62856c752a", + "3f188bbf2af01b25", + "3f22bd3eaef5d64d", + "3f4b6e4f245f6fab", + "3f8475de6ea558ac", + "3feda0b8b5637540", + "4057f5dbe4ffa4cd", + "40f62cd468a546cc", + "40fa0a4cb5193977", + "40fc6ad0aed92913", + "41bd2924e9afced5", + "420876ec1e5ba657", + "424f6d496746258b", + "4285d9b580f8a2a6", + "42bc15eb26cf2eec", + "42d2e68c29d9eef7", + "42f53e51476f1a3c", + "438b59086f6ebd64", + "43a4ca62d0c0b0da", + "43d9cb9d3fe30ff2", + "449c74a92ebb61a4", + "44becd7e128f4ba0", + "45098872448f5816", + "46459cfb9c1045d8", + "4750f5114dcaa60d", + "47913a485c3c8a18", + "4832a44c0df0bad2", + "4871a084d74a4bc0", + "4893140ae5daadaf", + "48a41f3f5110db1f", + "48d897f17754c7ce", + "48fd3d84b1a0bc68", + "491c766e35ae9f6e", "49e6665ee56d3bfc", + "4abc83fee59586b2", + "4b2f963b48e0f954", + "4b5521f85ab3fff7", + "4bf7d6ef245dd393", + "4c0e2794cb729825", + "4c90a96fb1ee09f6", + "4d21561351ecd00a", + "4d2a177384b23fd6", + "4d885e54f1361d04", + "4dbb108406acb1a5", + "4dc023ea2e3c4115", + "4dd0daec78093d31", + "4e29916c366fe997", + "4eb33712d4b16f43", + "4f3bf0c433b7f54f", + "4f8db13958be0f75", + "4f953ae00a24eb85", + "5103648897ba1b3a", + "5162627872566ada", + "51768783ab3e67f3", + "51a4bea9a77a55cf", + "51c589803c349e44", + "5346d5a4418466ab", + "53bba35f7ea6edc9", + "53fe111411edfa45", + "541da571bac5e279", + "5423483cd3c3b75a", + "5469d2b3e5fb8323", + "5498bfcecf19dfa3", + "54f754dc95137c47", "54fada9909a1edb3", - "8ae9a7fbbb50b6ed", - "61e7ec1c62704f39", - "f3bedaa5ffeeadb1", - "a3d152e25f8ce6c9", - "de2ce0c1b72d5c90", - "92ae986997c1d45c", - "29cc495628e76ffa", - "83bb8420c3cccf85", - "ddd9691cb063b07d", - "693325ca20a6325b", - "37b94c08f8e25249", - "26c7b173668167d8", + "5506b5232076d409", + "55669bcb3a9e09c0", + "55925bc39df20912", + "559e0907fef71cf6", + "55e52bcaab69d573", + "55e85aac6594c2fc", + "55f9f2fedfc7f069", + "568c473e6e5e95c7", + "56a56d39dbfe3516", + "56bca69ce012d4c1", + "56ceb1a9c003ac3f", + "56ebfd7a918891c6", + "5787ff8207fd751d", + "57e108dcd6ee072b", + "57e929939f1fd1c0", + "583c12b5bf2423c6", + "58688dfc2c59d3ab", + "587055293e9fa51d", + "58ece75f7f2b5099", + "58ef8ca403038960", + "596a0fd33f70da59", + "59820117bb7fa240", + "598e2a9978d3c79c", + "59b026d889aa0309", + "5a1764ef77908457", + "5b1b55f1997a9439", + "5b67a83de2aaa93a", + "5bf118ffa29520c0", + "5bfbc4a6cbab9eb1", + "5c5431cca026fb3e", + "5caf1c90ec7001b7", + "5cbd46653c3bea1a", + "5cec11774890d61e", + "5cfe68ed449e0478", + "5d4aae047020163b", + "5d674c06a559a5e7", + "5e4c757231135ba1", + "5e4e16da9d0630dd", + "5eb91b620bbf5bfc", + "5eda77c801e2df86", + "5edefd19948ad581", + "5ee9898421c06859", + "5efe98a1ba4df4d7", + "5f259ce91f0234a0", + "5f4cd1504f8b18e3", + "5f84cd800ee96468", + "5f99b00bf258f32f", "60ec8738947badb0", - "37fe6036c2ac932b", - "d69309b0a27c929c", - "01e5c6d31f763f56", - "adc6971cdb0613b0", - "ffc97a8fff838fa8", - "9ac6a8bda4fb7dc4", - "e247b50a301ce26c", - "e359ab2e436adf36", - "723ba544431f7874", - "fff289a66cb9ce4b", - "583c12b5bf2423c6", - "98a578f7758bb518", - "10233188161519ee", - "123c94418ddf1ad8", - "0bd651fcbb86f2a9", - "6a46ff1d1a5154b0", - "a5898123ec124b34", - "6fe5828936663aed", - "5506b5232076d409", - "1ea31bb5f6a15995", - "45098872448f5816", - "112fe9f123e277ee", - "09220515a7edf3f9", - "114e39b4da949e6d", - "621335449637e335", - "b82b48b0e915aaac", - "db9267bdc717c801", - "195366bfba7213a5", - "177dee4c1051e612", - "dcd67613012044b3", - "60f0dd83c8f32b8c", - "bff1330c13ccd99b", - "c06c7eb3afa4aea0", - "9ae7edbd0444982d", - "ec21d3c76c5ef65d", - "449c74a92ebb61a4", - "23a85a7f9c07412f", - "7942e835fb18c4ac", - "51c589803c349e44", - "f577607298c4b574", - "8944c4f14bbcaa74", - "f1a73b2232787b26", - "96c904c81aa2a241", - "0ef56cba2362e463", - "b965c3b6a03aefc4", - "35b9be2203d2b9e6", - "da6845ee99cfa275", - "977b04d04def60a7", - "7d3546a4000b5b75", - "82a2bd8fe4ed2087", - "135ca3602f0005bf", - "95774db1eb19b767", - "83020f91ce1aaf68", - "950ba1c7368c397f", - "98642c6f99438194", - "b7209eec8fe7fc58", - "b21be5dd6fdec83b", "60f08c85a518de1c", + "60f0dd83c8f32b8c", + "6136acacc653106c", + "61e7ec1c62704f39", + "61e862ed07d3cb03", + "621335449637e335", + "62313881916550ca", "62eb991153211de6", - "f9d9fe0dd9f7f110", - "017f3cea8275a3e1", - "b644b78aafc367b0", - "808de32aa819821d", - "cc99b5d3806b1e97", - "d07b6e3036abb8f1", - "db28c1d7f201af1a", - "997281c99ac93b3b", - "a596f2412c0f8fe2", - "dc51cf4e51f01086", - "692e91451741ef49", - "d3d257648d4c3105", - "d6d129c94c368155", - "f9a3871824bcd47a", - "d063a93beba2dd65", - "e4dc90de190437b8", - "3feda0b8b5637540", - "8da3e1e348d36e07", - "d905d620783d3768", - "bb509db67c18bc40", - "902dbd430adb77a5", - "cb9a110799bc37f8", - "41bd2924e9afced5", - "a2cbd066271b6b3b", + "6313a92bf426d2bb", + "6337d3dcc5362dc6", + "637657613004dbbd", + "63e55780fa1766de", + "64e80c255d51ff37", + "64ed7e55bdc50177", + "6529877d9799822a", + "654554952ecb20ac", "6587b26878da24b5", - "5498bfcecf19dfa3", - "2e997ba7beaed34d", - "0208e6b60a0b1a32", - "17bfc3d0570ab3d0", - "c459c68e95d54f87", - "0a6525cdbeb38df6", - "eac2041efb4cf9f7", - "a9f542d8e2be69fd", - "be9ca77bec616fe5", - "09f539aedefa9756", - "86e9da10bc176267", - "ad2351479ff6ee84", + "65a47f93451f13de", + "65af80e5c6538215", + "65fba51313f65938", + "6700028a309c3f7f", + "67240adc18be338d", + "67e75c6c6538110e", + "68875c05e00bd1aa", + "691b730a7bbb28b0", + "692e91451741ef49", + "693325ca20a6325b", + "696c193b3d13924e", + "6999db84bd629b32", + "69b877d99c360f97", + "6a46ff1d1a5154b0", + "6abef2102c86c0fc", + "6ae356d8abbfdeb9", "6b605ff743c859d0", - "59820117bb7fa240", - "f5a53fa6d2c3d9b2", - "9cc6c08cbaa694f2", - "5ee9898421c06859", - "93f14cad1cb201fa", - "183a5cd3e2aaa0ec", - "53bba35f7ea6edc9", + "6bdca9f13e19572e", + "6becc4c21730d391", + "6c2fc8308efeccb4", + "6c3e0fe95400f1f5", + "6cef8442092da1da", + "6d9c2bab8cc8d28c", + "6dde54b8c6d592ef", + "6df8f86449e56c05", + "6dfa24c82e89436b", + "6e9496b7254e02e2", + "6eacef3b2f976ce9", "6eeed6fefb1f0243", - "8b0204e041dd3eb8", + "6ef82dead8e3437f", + "6f13e8b708b43383", + "6f480809fc73b635", + "6f90b0bf6bbe884b", + "6fd73557577cf9ed", + "6fe5828936663aed", + "6ff89f3123a6a98f", + "70908a4cfdc5585c", + "7096b882b315807a", + "70f75a14390522b6", + "711a7b593ed41b02", + "71d3720cad53b289", + "71eae9be60b7fb6c", + "723ba544431f7874", + "731aac44217611ec", + "736ede0289146b59", "7386d04b20f354f4", - "a57845715134651a", - "2a91f42389fda90d", - "f7cbb33cc639b5b5", - "b5b8bd0cb8bb7658", - "86462bcec4492bfa", + "73dbc35c2b91c4fd", + "743318f3a2b4e281", + "74d685c3723bdf92", + "75609423e1f40c07", "75c08d873d0dcc06", - "6d9c2bab8cc8d28c", - "102bbdfcd696d7a9", - "bbc9a3a1d685d5f1", - "ce22219c2049985c", - "e2a45ce49bb2b731", - "21cffadbbb87c205", - "b4768c31e98b2170", - "16b0a0811bc4fe82", - "0ee132dec087b772", - "8bc3658bc958f81f", - "18b7e5bdf459ca13", - "f2516946c797b1a0", - "e56d5d27cb2b3e6f", - "0a6e50445a906052", - "33cada8c95bf55c5", - "ecee66b37d7576db", - "bc25a76284e7c7ce", - "e9c5d02aae54cdc5", - "b3855f3a6c06c01c", - "b24a14935edd51ad", - "69b877d99c360f97", - "f731aaaab07bdca2", - "7980d204dbc3ecee", - "8c6c4b4606608dfe", - "6e9496b7254e02e2", - "aadbb9992a672a1b", - "ca1fa7c3f26bb3c5", - "27ccec0740bb7915", - "ee4e169753085913", - "ea4f0c0a7b82f526", - "16f1a518cd842fcb", - "83c1ee8ab43ea00a", + "75ce95f62fc65fac", "76d52f916b4222b1", - "4d2a177384b23fd6", - "43a4ca62d0c0b0da", - "e23f9c02819688f6", - "16b24ff1bf3c079b", - "cd72fc2b62780a43", - "6eacef3b2f976ce9", - "fbdaf22c4f50391c", - "107c348f4ba0a41c", - "fbe41655a05a01ff", - "39c07ebbca541145", - "173704f0d2b8a02f", - "096b05114ee5e09f", - "a289b9d43310182b", - "cc6918e95b803734", - "5f84cd800ee96468", - "07ce7e4844f62c5c", - "22ede463dbf1a105", - "8ee7145a1b48b578", - "85fa17a19251a2ea", - "58ece75f7f2b5099", - "420876ec1e5ba657", - "2d0b4100c3ead88a", - "42f53e51476f1a3c", - "96c685a3cdd9ed59", - "eacc2cf0173eb438", - "9938be53082a02e6", - "d75ff490c057793c", + "781c243d0021d4c5", + "78a279b07a6d3367", + "78ef46252d580af0", + "793075287bf70a81", + "7942e835fb18c4ac", + "797ea41cc2e95f08", + "7980d204dbc3ecee", + "7a059d2754e088a3", + "7a26d92281df3e8e", + "7a5a4a9578503a55", + "7b63c31cfd0eb65f", + "7c04881c9a46b337", + "7c708f62e3280ec3", + "7c7e48e3aed8bfd4", + "7d3546a4000b5b75", + "7d4585a4b5b6e873", "7dfcefd1ffeaa941", - "131ec924c42ef90b", - "d3323745a51a604d", - "2eee0fdd5cdaf565", - "bdf8d5624e230986", - "7096b882b315807a", + "7e24168630d5124e", + "7e37ff763a97bc72", + "7e8763f553e43d12", + "7e92dcac20e009a5", + "7e9fc9f948e5cb17", + "7f02a580f5219377", + "8035f242b12cec6d", + "808de32aa819821d", + "80fe3ba3d9c9bf64", + "819d92a6492415d3", + "82a2bd8fe4ed2087", + "82dce1546c69369d", + "83020f91ce1aaf68", + "835211383d281d89", + "83b4d7d35a747d1b", + "83bb8420c3cccf85", + "83c1ee8ab43ea00a", + "84444ea4fabcc66c", + "84501b4439b66fd9", "84b1520706c3d784", - "b49efd086851f84d", - "e56e14e8cbf89e3c", - "3183669bf78db43b", - "c28289a45fc012a6", - "a14f221cf1761a8c", - "e7a197d6aefb66df", - "0349c893b81f7c1a", + "84c8bf1f809380bc", "84ff17001564ea47", - "b1401488f998c1f0", - "e367ecaf05dacedb", - "6f90b0bf6bbe884b", - "4abc83fee59586b2", - "e64266a9c13fc74e", - "37c8d23c44b95833", - "5edefd19948ad581", - "932cef2b9698dde8", - "158950ed4bc35274", - "4dc023ea2e3c4115", + "85fa17a19251a2ea", + "86462bcec4492bfa", + "86e9da10bc176267", + "86f1afdbbd29e48c", + "871066e9e2eaba52", + "8782c7c292ef2759", + "878af76320373ba0", + "87a80325171d8add", + "87abd998f3b7e26f", + "8817adf6eaf4f06c", + "8820852f4a1163b3", + "8825e2c99223c084", + "882a9d016c7e1f36", + "8865aed38c79d3fc", + "8870ddf70bf57890", + "8944c4f14bbcaa74", + "89d1f7ab8b77e41d", + "8a5f8686ab9de19e", + "8a7731e9eb363fb4", + "8ade0910a89717a8", + "8ae9a7fbbb50b6ed", + "8b0204e041dd3eb8", + "8b7145c13f36c324", "8bbbd89d09bb82ab", - "ac4687f4b15507b6", - "3392558f04e04663", - "64e80c255d51ff37", - "78ef46252d580af0", - "4b5521f85ab3fff7", - "6c3e0fe95400f1f5", - "988394e53c8e52d9", - "983d431922822165", - "0319b03323f257d1", - "3a3b0a9113835307", + "8bc3658bc958f81f", + "8c6c4b4606608dfe", + "8cfcf8dc64485a74", + "8cfd0c9f4bcd21b8", + "8d2c50e41a1a0061", + "8d68ee5c283ca6b0", + "8d92202f35f15712", + "8da3e1e348d36e07", + "8dcdbce3262bfcd6", + "8e713eb632acc712", + "8ee7145a1b48b578", + "8f3037c89c498e88", + "8f7cc0f456186028", + "8f7e22207cb3dd8b", + "8f81bb35e37f1b4c", + "902dbd430adb77a5", "91108cfcecb98297", - "b883e389d34b1bb1", - "af58ba642f4fc3d2", - "b0fd694500bfffab", - "d7501476836cf315", - "1cc05dbb992431b9", - "d4a1feebc1e1cb5d", - "ea316d47c03f9fdf", - "781c243d0021d4c5", - "aa3f466daa34d3ff", - "568c473e6e5e95c7", - "cdecc0eedcf05d2c", - "d3987535a95fbffd", - "f4da9e13977580f4", - "c483baec90d76587", - "6becc4c21730d391", - "7c04881c9a46b337", - "a93edea77d9f3338", - "c1e14f36f031a342", - "b12b4e4eacad1497", - "fd583001a87023fd", - "63e55780fa1766de", - "58ef8ca403038960", - "b9a6922899b66b8b", - "0ff82b1cc24b3105", - "0c9e215997d96f44", - "f0bd6a29eee94a2c", - "4f8db13958be0f75", - "c4f5a7748428e7d1", - "adc9df2489ce090a", - "e3231fac88242d81", - "c62b878b1ac43df9", - "a6c1dd30884a197c", - "5469d2b3e5fb8323", - "f588fbe6fd36bc6b", - "04c5a5a66fb7f89a", - "34ed76f0c32d9e93", - "6dfa24c82e89436b", - "d12a50d33b1c4253", - "ab88698a0c6ee1bb", - "13e5b88ad3becb77", - "cce214002aca738f", - "b6c3a9d53b9e784a", - "b1b04d862f4c32af", - "dbbb56b48acbaf22", - "adb678f7143014a0", - "e0999b85d2f77733", - "438b59086f6ebd64", - "6700028a309c3f7f", + "915e919390e6f58b", + "921ecc7c3e5f4884", + "9231adbf747903df", + "925da3b468003e72", + "9292ab2134e8a712", + "9299f6cd8a266541", + "92ae986997c1d45c", + "92c7355f5ea28e52", + "932cef2b9698dde8", "9337ab9573b366c1", - "4c0e2794cb729825", - "6f480809fc73b635", - "2760aeedce0b79db", - "bec58e970e8cd6c8", - "d3420c2bf8974a5d", - "57e929939f1fd1c0", - "65a47f93451f13de", - "b168db2651cbc1d6", - "75609423e1f40c07", - "0d4a95a8b52b787b", - "296618f10b019d9c", - "fce7e01f2cd0cac0", - "36641ce7192b5f92", - "4d21561351ecd00a", - "06b24bc51eb69188", - "dbe20d7c305b24eb", - "dbcf1311b3d95cdc", - "51a4bea9a77a55cf", + "93f14cad1cb201fa", + "940a18f62c79ffae", + "950ba1c7368c397f", + "953e132dd43ec03f", + "9545f2b0b8b92c8d", + "95774db1eb19b767", + "95c52fa37ef1eb3b", + "96106c2d31eea8b2", + "966a03f9f111aa35", + "96b0895b9f712d2f", + "96c685a3cdd9ed59", + "96c904c81aa2a241", + "975dae6e02765a90", + "977b04d04def60a7", + "97eb8171b51f9aad", + "97f302708066dc04", + "981f7ea91d7bdcb6", + "983d431922822165", + "98642c6f99438194", + "988394e53c8e52d9", + "98a578f7758bb518", + "9938be53082a02e6", + "994a2bf994b4facd", + "996691f0e03e303d", + "997281c99ac93b3b", + "9973eb31c46f8637", + "9ac6a8bda4fb7dc4", + "9ae7edbd0444982d", + "9b30bcdbccfd4c68", + "9ba1fa1ae8c3ae67", + "9be0c86102117436", + "9be76c10474e0c80", + "9cc6c08cbaa694f2", + "9d36f8ab4e1b45dc", + "9e55246befe96fb4", + "9e7c513fdd35df17", "9f400f89490fb8d5", - "55669bcb3a9e09c0", - "6337d3dcc5362dc6", - "55f9f2fedfc7f069", - "3d756c83419bba28", - "8f3037c89c498e88", - "2be10f342e68a89e", - "e501cafa3b025fb9", - "42bc15eb26cf2eec", - "05822f92f304fcd6", - "44becd7e128f4ba0", - "696c193b3d13924e", - "c5bc8b961a199646", - "2a07b85f61ac08b0", - "5cfe68ed449e0478", - "02c6fbcd90dcf05a", - "4285d9b580f8a2a6", - "16513a1bff79b46a", + "9f8c9a3aa64a9f8e", + "a00e93f2981ef418", + "a0621c7e36e9ecb1", + "a0fff0700364df49", + "a14f221cf1761a8c", + "a191b2da409f3518", + "a289b9d43310182b", + "a29d46ab6efee468", + "a2a5ccdf0846ce67", + "a2cbd066271b6b3b", "a334142059d865a2", + "a35ed74b7ea3ec4b", + "a3d152e25f8ce6c9", + "a465a767ceb7fd30", + "a49706f436a6873d", + "a4d0e44f4527180f", + "a5451ba38050a44d", + "a57845715134651a", + "a5898123ec124b34", + "a596f2412c0f8fe2", + "a5e8267322d0d7a6", + "a64e6bf63ab6a299", "a67919286833a7df", - "2078424be58e4db1", - "0f684159ad31f883", + "a6c1dd30884a197c", "a76054a284f0ce61", - "8870ddf70bf57890", - "15b121492722bffb", "a781ac2579798523", - "bf1338e9abe85c39", - "86f1afdbbd29e48c", + "a851c3b4377f4923", + "a9004216663480de", + "a925d8f1adc5cfe2", + "a93edea77d9f3338", + "a9f542d8e2be69fd", + "aa0213dd9f727219", + "aa3f466daa34d3ff", + "aadbb9992a672a1b", + "ab769495f97aab9c", + "ab88698a0c6ee1bb", + "ab8cf6af80a5c0e0", + "abef2b316c54f9e7", + "ac1257069b2f4afd", + "ac4687f4b15507b6", + "acee5f619f09bf79", + "ad2351479ff6ee84", + "adb678f7143014a0", + "adc6971cdb0613b0", + "adc9df2489ce090a", + "ae175aa839dbf692", + "ae9574428bd713e6", + "aeb79c7e573de0e5", + "aed7a612d966a786", + "af373587f2c57a24", + "af4c9b6e93062fde", + "af58ba642f4fc3d2", + "af78198822da03a7", "afcf1165734847de", - "f569fb2a6ddae452", - "882a9d016c7e1f36", + "afe441f40a42865e", + "b0fd694500bfffab", + "b12b4e4eacad1497", + "b1401488f998c1f0", + "b15c099c68671895", + "b168db2651cbc1d6", + "b1b04d862f4c32af", + "b1b385d5bad724ed", + "b21be5dd6fdec83b", + "b2436dcf1c8e5dc5", + "b24a14935edd51ad", + "b252ff42ce0b7443", + "b2ba4d07e03b47ea", + "b3855f3a6c06c01c", + "b4768c31e98b2170", + "b47b457f73c42c60", + "b49efd086851f84d", + "b4fc6a7b6c200b1b", "b584e424a8b3dabe", - "d80c26c2484dc87d", - "30e9ce0969941ffb", - "8cfcf8dc64485a74", - "d1df1bf926a69ca9", - "f621ca835cf98a6d", - "3519c14e4114d93d", - "db28c203846775d8", - "2d046178aa118fff", - "40fc6ad0aed92913", - "48d897f17754c7ce", "b5968cd571a14cf3", - "31fd07069aebf013", - "c18dc9f29967fe7f", - "afe441f40a42865e", - "f8a3fde3a557f44d", - "c8e381408b191a02", - "f9e9366287e79e4d", - "cc64f3b8666b17bf", - "e535b7a4343399e7", - "01ce6f422bec823d", - "e64eedbb40d09a50", - "ca88a34b0fcc5b0e", - "bd346dd3e46dff68", - "966a03f9f111aa35", + "b5985efd378af972", + "b5b8bd0cb8bb7658", + "b5b8de155e4e40e6", + "b644b78aafc367b0", + "b6a96b0dcf6eadf0", + "b6c15035f759ec50", + "b6c3a9d53b9e784a", + "b6e60b9025d116e4", + "b7132fe23fbdf9e1", + "b7209eec8fe7fc58", + "b764f5d5391e431e", + "b804c318fc0c1e9b", + "b82b48b0e915aaac", + "b883e389d34b1bb1", + "b887b7dcd6f1fdbb", "b95f2b009d70fad1", - "337ece90615ed69d", - "f14484e630df1dec", - "10af4a6378d0fb7a", - "5cec11774890d61e", + "b965c3b6a03aefc4", "b99c572f58cfb19e", - "b6c15035f759ec50", - "975dae6e02765a90", - "c96ac4c4015414ad", - "ea1b5ef156502d98", - "082380caaa153ee6", - "1ff08f79cb89c1de", - "87abd998f3b7e26f", - "13d2556ef392ed41", - "00a2b91e5b58cef1", - "cc5a3d2a52f99ab3", - "7a5a4a9578503a55", - "59b026d889aa0309", - "157f811757a0b29e", - "8825e2c99223c084", - "ed511caf8988dc45", - "04cb360502da6492", - "d317c9c2e53ce427", - "f9fb8170cecedc23", - "e3129aad84d5c40c", + "b9a6922899b66b8b", + "ba521df2c728c5a2", + "baead73ecc6e4a33", + "bb02df07a0d95e77", "bb21cecb8bbfdf63", - "46459cfb9c1045d8", - "102707a8c4d8a1b0", - "2bd9661657e78278", - "a191b2da409f3518", - "62313881916550ca", - "e5863682239df24b", + "bb509db67c18bc40", + "bbc9a3a1d685d5f1", + "bc1452fff95744b9", + "bc25a76284e7c7ce", + "bc715a45d7584d95", + "bd346dd3e46dff68", + "bdf8d5624e230986", + "be080cda1829826e", + "be9ca77bec616fe5", + "bec58e970e8cd6c8", + "bf0f48ffd0dff903", + "bf1338e9abe85c39", + "bff1330c13ccd99b", + "c06c7eb3afa4aea0", + "c10a3a62bf9fe782", + "c16f3dafab92df61", + "c1754dc2e7d666d8", + "c18dc9f29967fe7f", + "c1e14f36f031a342", + "c23218b044706800", + "c28289a45fc012a6", + "c3dc8cfd97735115", + "c459c68e95d54f87", + "c483baec90d76587", + "c497da3fdb2f5027", + "c4ca6565698f0e49", + "c4e6a655a15144dd", + "c4f5a7748428e7d1", + "c518dd59d12daf17", "c572cacbeb840621", - "e93cb94a06c6957e", + "c5bc8b961a199646", + "c621d1371b314913", + "c62b878b1ac43df9", + "c67724216ed5a318", + "c6c291cd525225b7", + "c8e381408b191a02", + "c8fc487eec95c21e", + "c902a788d699fe4a", + "c96ac4c4015414ad", + "ca1fa7c3f26bb3c5", + "ca63425a882729f3", + "ca88a34b0fcc5b0e", + "cb9a110799bc37f8", + "cbe5be4b63677c25", + "cc5a3d2a52f99ab3", "cc5e2f58b681e555", - "259e0ab1c4b690cf", - "fb2cfdfef4f93536", - "abef2b316c54f9e7", - "7f02a580f5219377", - "819d92a6492415d3", - "dd4cfd03d9b3f25f", - "996691f0e03e303d", - "5e4e16da9d0630dd", - "5162627872566ada", - "a465a767ceb7fd30", + "cc64f3b8666b17bf", + "cc6918e95b803734", + "cc932dbc293df39e", + "cc99b5d3806b1e97", "cce1a95634c6972b", - "7a26d92281df3e8e", - "109eb2158ca6a2f9", - "743318f3a2b4e281", - "06998c36415036aa", - "a49706f436a6873d", - "75ce95f62fc65fac", - "43d9cb9d3fe30ff2", - "4871a084d74a4bc0", - "8ade0910a89717a8", - "00b539d629e60965", - "74d685c3723bdf92", + "cce214002aca738f", + "cd72fc2b62780a43", + "cdbbbad526f0aa13", + "cdecc0eedcf05d2c", + "ce22219c2049985c", + "ce92195cf4a19084", + "cecb6a5d441fba98", + "cf1c544add4bf860", + "d063a93beba2dd65", + "d07b6e3036abb8f1", + "d07cacdb732c0b0f", "d08b5ddedd34448d", - "6999db84bd629b32", - "f7941427d5fa40ea", - "92c7355f5ea28e52", - "0f7ce013d15f6ff5", - "7b63c31cfd0eb65f", - "15fae84c1e6c6c57", - "4eb33712d4b16f43", - "67e75c6c6538110e", - "8817adf6eaf4f06c", - "97f302708066dc04", - "02cb3c385865ad2c", - "c6c291cd525225b7", - "f2e7c6dfd6a431c8", - "8820852f4a1163b3", - "ec8a2a25cdbda634", - "5bfbc4a6cbab9eb1", - "491c766e35ae9f6e", - "3f22bd3eaef5d64d", - "c497da3fdb2f5027", - "b6e60b9025d116e4", - "e89a1a7817dae7e4", - "15ae64b85b3e28d7", - "7e8763f553e43d12", - "691b730a7bbb28b0", - "8d92202f35f15712", + "d12a50d33b1c4253", + "d1df1bf926a69ca9", + "d1fe94ee32cbbf05", + "d2bece2b0eebce5b", + "d2d43918e5447c09", + "d317c9c2e53ce427", + "d32e5d4d5ff8b871", + "d3323745a51a604d", + "d3420c2bf8974a5d", "d34363168b1590bd", - "e6aaffd588f2832e", - "57e108dcd6ee072b", - "6529877d9799822a", - "384dca297bd4f1ca", - "b764f5d5391e431e", - "b5b8de155e4e40e6", - "d4647f5217d84ba3", - "dbd62ef3bd952bad", - "8dcdbce3262bfcd6", - "835211383d281d89", - "b7132fe23fbdf9e1", - "de50abaf95e79d34", - "c621d1371b314913", - "6ff89f3123a6a98f", - "5caf1c90ec7001b7", - "596a0fd33f70da59", - "40fa0a4cb5193977", + "d3987535a95fbffd", "d3c2a59a2cffa30c", - "9ba1fa1ae8c3ae67", - "598e2a9978d3c79c", - "08bff656ff61e949", - "0160f6becf137107", - "f67f3737a3b47208", - "c3dc8cfd97735115", - "541da571bac5e279", - "55925bc39df20912", - "af78198822da03a7", - "6f13e8b708b43383", - "5787ff8207fd751d", - "424f6d496746258b", - "2c41f9dfcd5014e6", - "e6dd3d435bd1b82e", - "a29d46ab6efee468", - "f4b4b026e87b5086", - "ae175aa839dbf692", - "97eb8171b51f9aad", - "a2a5ccdf0846ce67", - "aeb79c7e573de0e5", - "42d2e68c29d9eef7", - "f1300cb717ea9335", - "9231adbf747903df", - "e9845b4128afd538", + "d3d257648d4c3105", + "d4647f5217d84ba3", + "d4a14ea7adb375ed", + "d4a1feebc1e1cb5d", + "d5187fc1cd590073", + "d67edade78d25033", + "d69309b0a27c929c", + "d6ade0a63d0f9130", + "d6d129c94c368155", + "d71d0b082233758e", + "d7501476836cf315", + "d75ff490c057793c", + "d80c26c2484dc87d", + "d905d620783d3768", + "d9bfffa3cc6d1c53", + "da1139ecdc3512da", + "da2d3567a814d2d7", + "da6845ee99cfa275", + "db28c1d7f201af1a", + "db28c203846775d8", + "db9267bdc717c801", + "dbbb56b48acbaf22", + "dbcf1311b3d95cdc", + "dbd62ef3bd952bad", + "dbe20d7c305b24eb", + "dc51cf4e51f01086", + "dcd67613012044b3", + "dd4cfd03d9b3f25f", + "ddd9691cb063b07d", + "de2ce0c1b72d5c90", + "de50abaf95e79d34", + "e0283686d26420d9", + "e0999b85d2f77733", "e0be32a08b66d264", - "8f7e22207cb3dd8b", - "f68006f6c05e1211", - "40f62cd468a546cc", - "56ebfd7a918891c6", - "cf1c544add4bf860", - "c1754dc2e7d666d8", - "a4d0e44f4527180f", - "3a6a5f536bfbd4d4", - "d07cacdb732c0b0f", - "9973eb31c46f8637", - "925da3b468003e72", - "7c7e48e3aed8bfd4", - "24624a4e60ddedd4", - "3e182053b03f1faf", - "7a059d2754e088a3", + "e23f9c02819688f6", + "e247b50a301ce26c", + "e2a45ce49bb2b731", + "e3129aad84d5c40c", + "e3231fac88242d81", + "e359ab2e436adf36", + "e367ecaf05dacedb", + "e3a743ef04ecfb64", + "e4dc90de190437b8", + "e501cafa3b025fb9", + "e535b7a4343399e7", + "e56d5d27cb2b3e6f", + "e56e14e8cbf89e3c", + "e5863682239df24b", + "e5b04a0a3872954f", + "e64266a9c13fc74e", + "e64eedbb40d09a50", "e6959c2b66202cb8", - "b252ff42ce0b7443", - "5f4cd1504f8b18e3", - "be080cda1829826e", - "0f1b31ac93459d1f", - "ea17788d9a9d12eb", - "5103648897ba1b3a", - "b887b7dcd6f1fdbb", - "1494cff85b3dc2bb", - "2adb03f5294c31c2", - "5cbd46653c3bea1a", - "259df98e7476a843", - "c4e6a655a15144dd", - "9b30bcdbccfd4c68", - "84444ea4fabcc66c", - "01dc77d7f8f94d97", - "3ea02a62856c752a", + "e6aaffd588f2832e", + "e6dd3d435bd1b82e", + "e7a197d6aefb66df", + "e89a1a7817dae7e4", + "e8a37e48a8d34410", "e8adf99d340489ce", - "65af80e5c6538215", - "a00e93f2981ef418", + "e92ba97c2742e63d", + "e93cb94a06c6957e", + "e9845b4128afd538", + "e9c5d02aae54cdc5", + "ea17788d9a9d12eb", + "ea1b5ef156502d98", + "ea316d47c03f9fdf", + "ea330145d3ba738d", + "ea4f0c0a7b82f526", + "eac2041efb4cf9f7", + "eacc2cf0173eb438", + "ec21d3c76c5ef65d", + "ec8a2a25cdbda634", + "ecee66b37d7576db", + "ed511caf8988dc45", + "ee35d872179a1397", + "ee3a996c18ace68e", + "ee4e169753085913", + "ee6718f7a7e3d30b", "ef02bc67ebead2a2", - "9be76c10474e0c80", - "54f754dc95137c47", - "a0621c7e36e9ecb1", - "797ea41cc2e95f08", - "b4fc6a7b6c200b1b", - "fca98ecd003e1b82", - "a5451ba38050a44d", - "58688dfc2c59d3ab", + "f0bd6a29eee94a2c", + "f1300cb717ea9335", + "f14484e630df1dec", + "f1a73b2232787b26", + "f1b9158cc2b98388", + "f229a8078cb4ff82", "f2327302df1aa660", - "30c6777901ae346c", - "6fd73557577cf9ed", - "96b0895b9f712d2f", - "e92ba97c2742e63d", - "fb1fff87c87b82d6", - "122245dd15683e26", - "34506b8fab6dd7c3", - "8cfd0c9f4bcd21b8", - "5b1b55f1997a9439", - "09b07fbbb24e1b04", + "f2516946c797b1a0", + "f2e7c6dfd6a431c8", "f2eb0ef953f3f147", - "7e9fc9f948e5cb17", - "4893140ae5daadaf", "f2fd56c195593dd8", - "953e132dd43ec03f", - "1c8567a1b1a6c12a", - "48a41f3f5110db1f", - "032faef0207b715f", - "d2bece2b0eebce5b", - "8d2c50e41a1a0061", - "a9004216663480de", - "8f7cc0f456186028", - "10b17081dd7cad8f", - "6ae356d8abbfdeb9", - "1bcfaefe46abccf0", - "8f81bb35e37f1b4c", - "51768783ab3e67f3", - "b6a96b0dcf6eadf0", - "5b67a83de2aaa93a", - "4c90a96fb1ee09f6", - "cecb6a5d441fba98", - "48fd3d84b1a0bc68", - "00eb3e5c7a4537b1", - "3f188bbf2af01b25", - "7e37ff763a97bc72", - "3ba9f330ce93636b", - "21951e191e55b66e", - "ba521df2c728c5a2", - "ffe4328dcdc6b538", - "ae9574428bd713e6", - "ff0ffb9a32873433", - "56a56d39dbfe3516", - "a925d8f1adc5cfe2", - "7c708f62e3280ec3", - "aa0213dd9f727219", - "3334d22b6559d0e2", - "b2436dcf1c8e5dc5", - "8d68ee5c283ca6b0", - "4057f5dbe4ffa4cd", - "18a515a0fff0c0c3", - "4832a44c0df0bad2", - "c8fc487eec95c21e", - "8782c7c292ef2759", - "36df04ab9c978ab9", - "06e70fa70393d60f", - "559e0907fef71cf6", - "af373587f2c57a24", - "3746512864b941d4", - "1999d4ed82b3188e", - "793075287bf70a81", - "878af76320373ba0", - "ca63425a882729f3", - "95c52fa37ef1eb3b", - "d2d43918e5447c09", - "ab769495f97aab9c", - "2b0342d5b70b7a78", - "ee3a996c18ace68e", - "9d36f8ab4e1b45dc", - "3d8083b1e6c7b336", - "fa8384342c25ec59", - "373e7b1bc9066563", - "26c4912b140fde9f", - "d71d0b082233758e", - "9be0c86102117436", + "f34df0dd7011366e", + "f3bedaa5ffeeadb1", + "f4b4b026e87b5086", + "f4da9e13977580f4", + "f569fb2a6ddae452", + "f577607298c4b574", + "f588fbe6fd36bc6b", + "f5a53fa6d2c3d9b2", + "f621ca835cf98a6d", + "f67f3737a3b47208", + "f68006f6c05e1211", "f70d226e1598ac41", - "328fee1e52f7ac82", - "70f75a14390522b6", - "1188aebd8681e4fe", - "08408cbc394e7421", - "cc932dbc293df39e", - "d5187fc1cd590073", - "0b0648a9b9a26747", - "0cd406e7c5013b74", - "25c07c5d0a05d86c", - "9e55246befe96fb4", - "a0fff0700364df49", + "f731aaaab07bdca2", + "f7941427d5fa40ea", + "f7cbb33cc639b5b5", + "f8a3fde3a557f44d", + "f96bfb09bb1bd640", + "f9a3871824bcd47a", "f9d96d0eb2bc85cc", - "5eda77c801e2df86", - "ab8cf6af80a5c0e0", - "73dbc35c2b91c4fd", - "8e713eb632acc712", - "56bca69ce012d4c1", - "bc715a45d7584d95", - "bb02df07a0d95e77", - "5423483cd3c3b75a", - "6df8f86449e56c05", - "3e345d3c6002ad55", - "29f2dd90f57ea948", - "ee6718f7a7e3d30b", - "ea330145d3ba738d", + "f9d9fe0dd9f7f110", + "f9e9366287e79e4d", + "f9fb8170cecedc23", + "fa8384342c25ec59", + "fb1fff87c87b82d6", + "fb2cfdfef4f93536", "fb8010928a4bb56b", - "921ecc7c3e5f4884", - "940a18f62c79ffae", - "83b4d7d35a747d1b", - "b15c099c68671895", - "f1b9158cc2b98388", - "8b7145c13f36c324", - "37e0d3cb098458d0", - "080aa6966d757f05", - "ee35d872179a1397", - "6abef2102c86c0fc", - "096ccb6bcb3d2731", - "71d3720cad53b289", - "15ee7651b7ad610c", - "0da9d81c9f563df5", - "6ef82dead8e3437f", - "fd966f807d3ba9fe", - "b804c318fc0c1e9b", - "4750f5114dcaa60d", - "9e7c513fdd35df17", - "b2ba4d07e03b47ea", - "cbe5be4b63677c25", - "53fe111411edfa45", - "87a80325171d8add", - "3f8475de6ea558ac", - "a64e6bf63ab6a299", - "c10a3a62bf9fe782", - "711a7b593ed41b02", - "5a1764ef77908457", - "6c2fc8308efeccb4", - "1ac8de53c0894d18", - "bf0f48ffd0dff903", - "a851c3b4377f4923", - "f96bfb09bb1bd640", + "fbdaf22c4f50391c", + "fbe41655a05a01ff", + "fca98ecd003e1b82", + "fce7e01f2cd0cac0", "fd3f988b7a505c45", - "84c8bf1f809380bc", - "c16f3dafab92df61", - "55e85aac6594c2fc", - "6cef8442092da1da", - "1172a88e009dc248", - "af4c9b6e93062fde", - "96106c2d31eea8b2", - "c518dd59d12daf17", - "871066e9e2eaba52", - "acee5f619f09bf79", - "0fe0bb1054f49b7f" + "fd583001a87023fd", + "fd966f807d3ba9fe", + "ff0ffb9a32873433", + "ff45605151ec4eb1", + "ffc97a8fff838fa8", + "ffd213e55eee8cc8", + "ffe4328dcdc6b538", + "fff289a66cb9ce4b" ] \ No newline at end of file diff --git a/src/sec_certs/data/reference_annotations/split/train.json b/src/sec_certs/data/reference_annotations/split/train.json index b2e65750..bc52e9b3 100644 --- a/src/sec_certs/data/reference_annotations/split/train.json +++ b/src/sec_certs/data/reference_annotations/split/train.json @@ -1,466 +1,532 @@ [ - "412316e54e63beef", - "ce3ee1fa409676b3", - "cb862d0020c32547", - "bda89f3b2429ab6a", - "388653542ec9fb6c", - "7f1f7bffe46d4c85", + "00a2b91e5b58cef1", + "00ad121fa63e5dfb", + "00b539d629e60965", + "00eb3e5c7a4537b1", + "00efeb17bcaafce6", + "0167c92c0d8c8b47", + "017f3cea8275a3e1", + "01ce6f422bec823d", + "01dc77d7f8f94d97", + "01e5c6d31f763f56", + "01e8805514b4ef67", + "0208e6b60a0b1a32", + "0213bdb5ebb8dc19", + "02c6fbcd90dcf05a", + "02cb3c385865ad2c", + "0319b03323f257d1", + "03281d5d8a77be56", + "032faef0207b715f", + "0349c893b81f7c1a", + "037577fc2019fcfa", + "03aded94fb04c62e", + "04c5a5a66fb7f89a", + "04cb360502da6492", + "054bd8196e015197", + "055985699aee1e09", + "05726637bd47a762", + "05a8a84b60a30ceb", + "05c03233d710cf54", + "05d96aada7bb839c", + "06b1a65aa87ffa78", + "06b24bc51eb69188", + "06e505abb8dad1b8", + "06e70fa70393d60f", + "075ddd00e07fa7ed", + "07ce7e4844f62c5c", + "080aa6966d757f05", + "082380caaa153ee6", + "0832672073492f50", + "08408cbc394e7421", + "09220515a7edf3f9", + "096b05114ee5e09f", + "096ccb6bcb3d2731", + "09963e79b21cde12", + "09b07fbbb24e1b04", + "09b1192474bc8b27", + "09b17cb9b3c8b1bb", + "09be49631855ae19", + "09f539aedefa9756", + "0a6e50445a906052", + "0aad0176e35b3749", + "0ac0120f667a8dcf", + "0b0648a9b9a26747", + "0b3e1cdf3ef9413d", + "0baac042dac42abc", + "0bd651fcbb86f2a9", + "0bd759633c0ceec6", + "0bf7a19b22163465", + "0c9e215997d96f44", + "0cd406e7c5013b74", + "0ce083454814c1ba", + "0d08a211bd29f963", + "0d164fb0d72dfa00", + "0d4385899adc0781", + "0d4a95a8b52b787b", + "0d730f18ae9d5694", + "0da9d81c9f563df5", + "0ee132dec087b772", + "0ef56cba2362e463", + "0eff4063e68d7cd6", + "0f040d1ec658b8a5", + "0f1b31ac93459d1f", "0f1bd208c27d202c", - "b8370ceababbca40", - "12e20466a0f08342", - "9b8b9b7fcd97bc60", - "32e083734907ad15", - "294ef6cb28408ec9", - "f17fe9ceea628f62", - "6f38b9bce48590e3", - "b61363e51ed90c7a", - "9328c028356ab522", + "0f3900cdcd0c7f3e", + "0f684159ad31f883", + "0fe0bb1054f49b7f", + "0ff82b1cc24b3105", + "0ff9337564f4f742", + "10107b83e7393c50", + "10233188161519ee", + "102707a8c4d8a1b0", + "102bbdfcd696d7a9", + "107c348f4ba0a41c", + "10868774b23c5730", + "109eb2158ca6a2f9", + "10af4a6378d0fb7a", + "10b17081dd7cad8f", + "112fe9f123e277ee", + "114e39b4da949e6d", "11687f8008b8198d", + "1172a88e009dc248", + "1188aebd8681e4fe", + "11a98c5d5ff6f40f", + "122245dd15683e26", + "123c94418ddf1ad8", "12c3b8b612c581cb", - "db3b58c1836b220e", - "f87b5bc03e18d010", - "a3de319045809828", - "ed22d4c0f09c3e3a", - "09b1192474bc8b27", - "d4e1874210b91dd4", - "c9b0cae8b5784d2d", - "dfe9de5d26e08949", - "97d00a9f198a6e57", - "d75282fdda80b6fd", - "6d41460480570d22", - "a7666a5157a31d32", - "5c3806bf79eeab7f", - "d2d0b2521a1f186d", - "6dfe436482047387", - "7e426c62da58175b", - "9dbb21687510991c", - "2b2dd14fee46498d", - "cf32640f396edb87", - "4d5cb991d2675a6a", - "751a324c3f87f58a", - "8bb25a14b21fcce6", - "243291b6c7ecde61", + "12e20466a0f08342", + "12fd6b1b6c4aecf3", + "131ec924c42ef90b", + "1357546ec01b1a65", + "135ca3602f0005bf", "13a92431f9683226", - "7f4d3b659fc17c09", - "dba4b5a166f9456a", - "768dda4099b6f2d9", - "668b5c95eb0229c1", - "1b797b27b981aafe", + "13d95239226aa537", + "13e5b88ad3becb77", + "1416afc51d8ce022", + "1468948b4f0509ab", + "1494cff85b3dc2bb", + "14df54305cad83b4", + "14e053dfe9057aa8", + "157b741a0f529a0a", + "157f811757a0b29e", + "158950ed4bc35274", + "15d68159595eae09", + "15d935e36152eb15", + "161216fcc9eb8d34", + "16bdbde359584f99", + "17a23970b35a4f44", "17b5257b3755a07c", + "185aa4f76ff15d62", + "188d3cda4e6f2d5d", + "18bb29b147e80caa", + "18e68a485e648ddc", + "19c60923e31777e7", + "19d14dec2d11fd96", + "19f06ccd7136a2b4", + "1a191f3535ccaea3", + "1b0a6ecee1f830fc", + "1b797b27b981aafe", + "1b8705a0486b3be1", "1c34d31598257da8", - "775b1fdecba5cb81", - "e0265f0fb8e196c0", - "161216fcc9eb8d34", - "7574381766e293c7", - "9529eb793550093c", - "4960693ee45f2ae6", - "cacdfdafce47678c", - "60f3bd10ee9be85b", - "4f5f41ecf7517e63", - "c7fc146c5ccf4821", + "1c9bd4db3d8388f7", + "1ca9b79a98ef45fa", + "1d89cece70c6cb64", + "1e40bd733acd56bc", + "1e58fd6b31d72e2a", + "1eabf957a9401fab", + "1ee7ecee9e7e131c", + "1ee8a6a4d8f26d65", + "1f9f4b843070fc1b", + "1fa2284c3dcf136a", + "1fb1564dfb0f0b04", + "1ff5d648a625f3c7", + "21c3236216f620ff", + "21d71bac070403e0", + "22dd52fbce13f27e", + "23ce630f1ab642b8", + "242fdec5d0ed5010", + "243291b6c7ecde61", "2477ebdeb9bcf4aa", - "6ca1063d82ab795a", - "d2a4b8cb9ae7fe8f", - "c2024b43eeaa96de", - "a6fe8fe0aaf2fa92", - "815d67f8292ad7c3", - "19c60923e31777e7", - "97d5f9e6e0a11e37", - "c39e6ef4060594e2", - "a76d1c0d9964d583", - "a52f807db0eb7fdb", - "6e4de0c194952a21", - "317713948b4473b3", - "73e4f65789159996", - "54375fc2889bdfb4", + "253e4d53f952a882", + "25eade3365f92578", + "2674228d88c4f3d6", "2731d8ddba404fad", - "617810f97f3f0b4f", - "e8e5f42a4e4005c7", - "03aded94fb04c62e", - "1fb1564dfb0f0b04", - "e7e366f3619b5953", - "cd43abdf668dbb23", - "1fa2284c3dcf136a", - "c96343baf608174d", - "c32378a010479b33", - "99223aca5d9eb3b3", - "678b2b51e429444a", - "727bce62137f97b7", - "62bf5de838cd3a7c", - "732271484650d27a", - "b42009eb34746731", + "27b4366a0f1fcdef", + "27d11629261d8806", + "28e70f46ba4394dd", + "294ef6cb28408ec9", + "29b0321ec6b75ebd", + "29c65bb00fb9aa6c", + "2b2dd14fee46498d", "2b944ab6ce8fe8e1", - "8ff91b407ccd4b18", - "8fbe28bc70a821bd", - "86255c61e33c2caf", + "2bb4685a03857b2c", + "2c15dfc106ebf8d8", + "2c4caf71ca3735a8", + "2d2ce200fea72359", + "2e2d56abe9dd6e22", + "2ebb5bdbe7e00a78", + "2f418fb3ced3820a", + "30120e4f3aa2f30a", + "30f71b100c5cebae", + "317713948b4473b3", + "32431779b1c54c0a", + "32e083734907ad15", + "335800e03030fb93", + "34370b67b5e675c3", + "3515801dee00995f", + "366786e98797ebf3", + "36ed04f4b45e3ab9", + "371dca18821f7714", + "376b8eb06fc0b40a", + "388653542ec9fb6c", + "388b6899fb38b762", + "3897d8d16ba333e8", + "3b0c5666e54a2394", + "3cdca4cc05854ade", + "3da6e0f0f97b3d2f", + "3db023c27932ad93", + "3ebe69af702fa116", + "3ef6d59aeaa6d26e", + "3f45ee333ce457bb", + "3f87aa90c0aaf83b", + "41148b8edaf1106e", + "412316e54e63beef", + "41fb9717f70d3037", + "42799014c183e1b3", + "42a062fe7f1ab253", + "446aa68e0c4c5083", + "44f1f74c09e634cb", + "4526aa14337a2b0c", + "454f266801ae2ae0", + "457fcec1ad31841f", + "463ecd64b7506048", + "4761b0a9b046cc66", + "4783477a22587a6e", + "4960693ee45f2ae6", "49bf2cf97dd4ca29", - "9776cff7d7ead7bb", - "69347104bc7e740a", - "818f62b997d143e3", - "bc17c54936484a90", - "824a8da5564344a3", - "fe223a9f01625d62", + "4a6100f1ddaef93b", + "4bede5d2af4cb11f", + "4c1ccde478ce063a", "4c3282bd1640fcbc", - "ee1c6dd97918d74a", - "ace99e269d2a1bd9", + "4d5cb991d2675a6a", + "4d776db86abecdda", "4f3bf2d41fc130df", - "ca5da2fe138af656", - "41fb9717f70d3037", - "de9a98c311683ef3", - "bc260c38202644f4", - "30120e4f3aa2f30a", - "e66a9b22df7fe32d", + "4f5f41ecf7517e63", + "4fd2ed9e6088176e", + "502645d74f607b06", + "536aac44b608f951", + "54273dd266fce692", + "54375fc2889bdfb4", + "543d800fd468f0fc", + "54dd28994604ee35", "559a5c3c4c23a9d3", - "3515801dee00995f", - "f20be2311365981c", - "e10bc3422afaec8a", + "5682e68143deea61", "5734ff91ea85611b", - "dabbb27e6faa13d5", - "7a8d4ed693d443d7", - "d94b13d95fd1c12a", - "e2e705cabd42e40e", - "b434dea9be2868db", - "e9e12b044b495c6f", - "a07548a64fac5794", - "5df7b3241b307f8b", - "18e68a485e648ddc", - "01e8805514b4ef67", - "13d95239226aa537", - "3f87aa90c0aaf83b", - "f2dc875bd399a806", - "a83fe0ea2f391ee8", - "7793c695818500d2", - "a38fbb576027f6c6", - "2f418fb3ced3820a", - "b5b0e3205923e723", - "627bfd69c2a831cf", - "e0b122da55f1f002", - "f6579adcbf5faa99", - "b33eaac3aae2f1e7", - "e6e8add5e4db2d9d", - "b0b7b073ca2dfe5f", - "fa67fbbcfc0b9442", - "af61a31e3fd0d6f0", - "fe445dc1c13738d6", - "0d08a211bd29f963", - "f02cf5078e4548e0", - "a276bd58a15d5be6", - "9d0a023ecd4ad7c5", - "0ff9337564f4f742", - "fbb507bc72376d36", - "6ddd0d9c37f27706", - "6a2323c5267ae0b7", - "744a7a202d909323", - "714559f7e1bf956c", - "c92a196d0408a5ff", - "f869956d14ea1694", - "34370b67b5e675c3", - "388b6899fb38b762", - "dd81eae9478576f3", - "8975cbb58ab67846", - "055985699aee1e09", - "1416afc51d8ce022", - "eea87713e11a55c2", - "44f1f74c09e634cb", "598b6c745cdcd404", - "e9b7e07dc5598afe", - "0b3e1cdf3ef9413d", + "59f15180eddee8b2", + "5a12431e371f3ce9", + "5a66027d34aafc52", + "5bf4e37462a1b715", + "5c3806bf79eeab7f", + "5cc6ace5ef0d7525", + "5cdef03a3004a6ff", + "5deb0179858d78a1", + "5df7b3241b307f8b", + "5e5d4b87195388c7", + "5ee0fb9826fec164", + "5f1bc82727f54376", + "5f3c4c189362c2e5", + "5f4b39feb2a82dac", + "5f7760ca5dd8b8ef", + "600e2a1d0f05afd9", + "608fe49756dc68cc", + "60b45a1573fc53e9", + "60f3bd10ee9be85b", "610e118d804e4076", - "fb9ec9e846ea4e87", - "aa096ffde94b953b", - "887afbb0843425b6", - "e2275acc4f1c2d0d", - "b3b7ac7aae87793d", - "95b2a54b81b0bc6c", - "77acb3a4a3f58e0b", - "ecfb020c517d37c1", - "9302a95ca3efba5c", - "ced6075dd1813122", - "ab3af998dff7a2ef", - "0f3900cdcd0c7f3e", - "d699ed2b1adc6be4", - "1e40bd733acd56bc", - "aea11fd4d2a7709d", - "cf7705e45048032e", - "1d89cece70c6cb64", - "f73d08f8cdfe276d", - "cfdacd53c732343c", - "9a94525307a1b13d", - "543d800fd468f0fc", - "25eade3365f92578", - "e886bfa3a4f4ffd4", - "b8437dbfeeaf0f04", - "05c03233d710cf54", - "18bb29b147e80caa", - "10107b83e7393c50", + "617810f97f3f0b4f", + "61832cb4291c343f", + "61bff064f593ecc0", + "623dee827372d496", + "627bfd69c2a831cf", + "6291bcc06184d125", + "62bf5de838cd3a7c", + "63d0bacbab451d2d", + "659f501baa5997da", "65ab2fd5467ef78d", - "10868774b23c5730", - "253e4d53f952a882", - "a47713a44e4c656b", - "f2e00a1fee27bdd7", - "4783477a22587a6e", - "15d935e36152eb15", - "4a6100f1ddaef93b", + "668b5c95eb0229c1", + "66e47f5d28e2f95b", + "678b2b51e429444a", "680aeb0a20a9fed3", - "14df54305cad83b4", - "188d3cda4e6f2d5d", - "fc373293fb7c56ac", - "79f48541164e16ff", - "e9fc2f8b110e71a0", - "ce14eb1b2318f054", - "12fd6b1b6c4aecf3", - "5f4b39feb2a82dac", - "0f040d1ec658b8a5", - "09be49631855ae19", - "c8ff17063166050d", - "76b289d10702db4d", - "eac7be8886cb755b", - "84c868322995b179", - "a515a81bc4be3cd2", - "b2bab86575d21b44", - "0eff4063e68d7cd6", - "32431779b1c54c0a", - "c118ee1ff2d1bede", - "1b0a6ecee1f830fc", - "cd052c606f2f9124", - "5deb0179858d78a1", - "dd1472476dfe4a8b", - "1a191f3535ccaea3", - "a9d336b90e8b94a2", + "691119fe8a2f8e1c", + "69347104bc7e740a", + "6962629b904be54d", + "6a2323c5267ae0b7", + "6a5cb89e80f86ce9", + "6a999675c9422dfb", + "6b003152c5459126", + "6b249c38b49ec9f0", + "6c14f733fe7aa28e", "6c222ea916b4f5e1", - "c5a60dbdb668bc10", - "f3a461d6bf980196", - "977477f3f0bf1bfc", - "1ee7ecee9e7e131c", - "d1a87288683080c4", - "185aa4f76ff15d62", - "23ce630f1ab642b8", - "7e9dd6cb86b58f95", - "b30ef2c1aaec9236", - "a154a01ff4cf04c7", + "6ca1063d82ab795a", + "6d41460480570d22", + "6d6ade44dcc497dd", + "6dc47e3f5d35aedd", + "6ddd0d9c37f27706", + "6dfe436482047387", + "6e4de0c194952a21", + "6eb6b29c45f6355f", + "6f38b9bce48590e3", + "6f8d7a6a1dea6a3a", + "714559f7e1bf956c", + "725b6b95cd47b601", + "727bce62137f97b7", + "732271484650d27a", + "73e4f65789159996", + "744a7a202d909323", + "751a324c3f87f58a", + "7574381766e293c7", + "76610bf4610246c7", + "768dda4099b6f2d9", + "76b289d10702db4d", + "775b1fdecba5cb81", + "7793c695818500d2", + "77acb3a4a3f58e0b", "79c729d80cb73939", - "a9639521d8a5ab09", - "f32a1e7e2843c096", - "8c21cafc192eeea5", - "59f15180eddee8b2", - "2ebb5bdbe7e00a78", - "bd1a21ea3d047c14", - "dbf15cc4f389196f", - "98f6f9dbb1540a31", - "cb203cf5d91b1ae3", - "f921d39906e6d93d", - "6c14f733fe7aa28e", - "f71fca5fc684df8b", + "79f48541164e16ff", + "7a8d4ed693d443d7", + "7a9f073396eaec67", + "7d8dc0575933f142", + "7e24a5a087db6814", + "7e426c62da58175b", "7e82a1ceb29e4019", - "54273dd266fce692", - "536aac44b608f951", - "921e042759b30033", - "0167c92c0d8c8b47", + "7e9dd6cb86b58f95", + "7f1f7bffe46d4c85", + "7f4d3b659fc17c09", + "7f9eb1b8217d15ad", + "815d67f8292ad7c3", + "818f62b997d143e3", + "824a8da5564344a3", + "82b7b915b7af9c17", + "845bb039719ac5d8", + "84a75b6fc33b669e", + "84c868322995b179", + "85054d3f47bee56c", + "850f5b2c312299d5", + "86255c61e33c2caf", + "882df28aee338320", + "887afbb0843425b6", + "890f788a91fc2fa1", + "8975cbb58ab67846", + "8ba22f6c9651edc3", + "8bb25a14b21fcce6", + "8bdb610131555c12", + "8c21cafc192eeea5", + "8ecc2cf269498bee", + "8fbe28bc70a821bd", + "8ff91b407ccd4b18", "90581f56f6846be5", - "a91ca8347f800ee7", - "dc278166b5d14512", "90c290ed4ded479d", - "cee70006132a6f46", - "1357546ec01b1a65", - "5a66027d34aafc52", - "2c4caf71ca3735a8", - "0baac042dac42abc", - "54dd28994604ee35", - "60b45a1573fc53e9", - "6a999675c9422dfb", - "0d164fb0d72dfa00", - "b24f997d9358b324", - "df6728b8420ab3b4", - "d50ab8fd0c085fb6", - "9ae8bef18938d2df", - "19d14dec2d11fd96", - "f0ce40dfe333e918", - "3da6e0f0f97b3d2f", - "0d730f18ae9d5694", - "ecba01acba8df2ec", - "17a23970b35a4f44", - "1b8705a0486b3be1", - "1ca9b79a98ef45fa", - "66e47f5d28e2f95b", - "5f1bc82727f54376", + "921e042759b30033", + "9302a95ca3efba5c", + "9328c028356ab522", + "9529eb793550093c", + "95b2a54b81b0bc6c", + "95c69029642bdb41", + "9650ee69410c68ec", "9743174f44c5e2c9", - "ec84060a194fd9cc", - "845bb039719ac5d8", - "ea6d47427a2bb349", - "27d11629261d8806", - "6a5cb89e80f86ce9", - "29b0321ec6b75ebd", - "d355c778fcfee3a9", - "0ce083454814c1ba", - "c06d86db280f9b2b", - "b0e95af7c706ffa0", - "b9a973281ea252b3", - "d0705c9e6fbaeba3", - "a3210e75e10a3ba8", - "61832cb4291c343f", - "22dd52fbce13f27e", + "977477f3f0bf1bfc", + "9776cff7d7ead7bb", + "97d00a9f198a6e57", + "97d5f9e6e0a11e37", "985ffde955eee01a", - "882df28aee338320", - "b23babf61349db1a", - "1c9bd4db3d8388f7", - "1ff5d648a625f3c7", - "691119fe8a2f8e1c", - "c30de3192d2e8ec2", - "5bf4e37462a1b715", - "de23f9f77f9b49f0", - "dadd7dd2d4247627", - "36ed04f4b45e3ab9", - "6b249c38b49ec9f0", + "98f6f9dbb1540a31", + "99223aca5d9eb3b3", + "9a22a2bb323cfd0f", + "9a94525307a1b13d", + "9aa4b513807ea22c", + "9ad46ff913bb805f", + "9ae8bef18938d2df", + "9b8b9b7fcd97bc60", + "9c6440ecec046cc0", + "9d0a023ecd4ad7c5", + "9dbb21687510991c", + "9ef6037ebd29a9be", + "a07548a64fac5794", + "a154a01ff4cf04c7", + "a276bd58a15d5be6", + "a3210e75e10a3ba8", + "a38fbb576027f6c6", + "a3de319045809828", + "a47713a44e4c656b", + "a515a81bc4be3cd2", + "a52f807db0eb7fdb", + "a55ace0305e39654", "a69ec271f8651d1b", - "7f9eb1b8217d15ad", - "6b003152c5459126", - "6eb6b29c45f6355f", - "502645d74f607b06", - "6d6ade44dcc497dd", - "3ebe69af702fa116", - "29c65bb00fb9aa6c", + "a6cdbfd0a7bef417", + "a6fe8fe0aaf2fa92", + "a7666a5157a31d32", + "a76d1c0d9964d583", + "a7d6e1aeb5e009f0", + "a83fe0ea2f391ee8", + "a867d281d34d34b7", + "a904d6cb272e8533", + "a91ca8347f800ee7", + "a92a031f28de3544", + "a9639521d8a5ab09", + "a9d336b90e8b94a2", + "aa096ffde94b953b", + "aabc3155ba9aec95", + "ab3af998dff7a2ef", + "ace99e269d2a1bd9", + "aea11fd4d2a7709d", + "af61a31e3fd0d6f0", + "b07adba37aa1b7e2", + "b0b7b073ca2dfe5f", + "b0e95af7c706ffa0", + "b10ee0bb166cfd86", + "b23babf61349db1a", + "b24f997d9358b324", + "b26c9f300c14239d", + "b2bab86575d21b44", + "b30ef2c1aaec9236", + "b33eaac3aae2f1e7", + "b3b7ac7aae87793d", + "b42009eb34746731", + "b434dea9be2868db", "b4a2999b439b1ba8", - "3b0c5666e54a2394", - "8bdb610131555c12", - "b10ee0bb166cfd86", - "e676143c80802a59", - "15d68159595eae09", - "06b1a65aa87ffa78", - "5ee0fb9826fec164", - "2c15dfc106ebf8d8", + "b5b0e3205923e723", + "b61363e51ed90c7a", + "b8370ceababbca40", + "b8437dbfeeaf0f04", + "b9482c3bd09d03dd", + "b9a973281ea252b3", + "b9d8c4cb5cf8a4d6", + "baa8b8c5f52810af", "bab97726875c0f14", - "a867d281d34d34b7", - "eeb2e28f54484dc9", - "d753660b3c2bdd84", - "5f7760ca5dd8b8ef", - "8ba22f6c9651edc3", - "9650ee69410c68ec", - "608fe49756dc68cc", - "e7bfc160fd3c0d5d", - "335800e03030fb93", - "446aa68e0c4c5083", - "1e58fd6b31d72e2a", - "4c1ccde478ce063a", - "5cc6ace5ef0d7525", - "09b17cb9b3c8b1bb", "bbf0ceb1e0e4c6ee", - "c78ba3edcbd57788", - "ead1076787fdd7e4", - "8ecc2cf269498bee", - "d6d6b7c041820ab8", - "d837b4fde544a9f2", - "3ef6d59aeaa6d26e", - "fca2af05ee78ac5c", - "cf5cfcae47e5e3a1", - "baa8b8c5f52810af", - "3f45ee333ce457bb", - "7e24a5a087db6814", - "30f71b100c5cebae", - "9a22a2bb323cfd0f", - "84a75b6fc33b669e", - "21d71bac070403e0", - "0d4385899adc0781", - "05726637bd47a762", - "7a9f073396eaec67", - "5cdef03a3004a6ff", - "7d8dc0575933f142", - "890f788a91fc2fa1", - "5682e68143deea61", - "6dc47e3f5d35aedd", - "16bdbde359584f99", - "0bf7a19b22163465", + "bc17c54936484a90", + "bc260c38202644f4", + "bd1a21ea3d047c14", + "bda89f3b2429ab6a", "bdd16afe92166295", - "9ad46ff913bb805f", - "b07adba37aa1b7e2", - "3cdca4cc05854ade", - "41148b8edaf1106e", - "85054d3f47bee56c", - "aabc3155ba9aec95", - "dbaf9789160ca414", - "463ecd64b7506048", - "cef1c9957f4ff612", - "1ee8a6a4d8f26d65", - "a92a031f28de3544", + "c06d86db280f9b2b", + "c118ee1ff2d1bede", + "c175537338906951", "c1a87780d88d85d9", - "659f501baa5997da", - "95c69029642bdb41", - "e10d907c64d75e3a", - "4fd2ed9e6088176e", - "1eabf957a9401fab", - "cf8ac5972027dda5", - "623dee827372d496", - "f041e5b526e79ef4", - "0832672073492f50", - "0213bdb5ebb8dc19", - "d49988efd778ca9d", + "c2024b43eeaa96de", "c301c38902477230", - "376b8eb06fc0b40a", - "037577fc2019fcfa", - "14e053dfe9057aa8", - "28e70f46ba4394dd", - "b9482c3bd09d03dd", - "d2b0cb5a911b8ef8", - "05a8a84b60a30ceb", - "63d0bacbab451d2d", - "5f3c4c189362c2e5", - "c7d162e62f174cd5", - "1f9f4b843070fc1b", - "850f5b2c312299d5", - "b9d8c4cb5cf8a4d6", - "f6147b5c0a147123", - "3db023c27932ad93", - "42a062fe7f1ab253", - "a904d6cb272e8533", - "e86db251be2c8a60", - "5e5d4b87195388c7", - "9ef6037ebd29a9be", - "242fdec5d0ed5010", - "0bd759633c0ceec6", - "6f8d7a6a1dea6a3a", - "a55ace0305e39654", - "19f06ccd7136a2b4", - "2e2d56abe9dd6e22", + "c30de3192d2e8ec2", + "c32378a010479b33", + "c3549567b1ad7bdf", "c385c0ffaa17bd29", "c38734a9eeea0ff8", - "3897d8d16ba333e8", + "c39e6ef4060594e2", + "c5a60dbdb668bc10", + "c78ba3edcbd57788", + "c7d162e62f174cd5", + "c7f6cb518fb19151", + "c7fc146c5ccf4821", + "c8ff17063166050d", + "c92a196d0408a5ff", + "c96343baf608174d", + "c9b0cae8b5784d2d", + "ca5da2fe138af656", + "cacdfdafce47678c", + "cb203cf5d91b1ae3", + "cb862d0020c32547", + "cd052c606f2f9124", + "cd43abdf668dbb23", + "ce14eb1b2318f054", + "ce3ee1fa409676b3", + "ced6075dd1813122", + "cee70006132a6f46", + "cef1c9957f4ff612", + "cf32640f396edb87", + "cf5cfcae47e5e3a1", + "cf7705e45048032e", + "cf8ac5972027dda5", + "cfdacd53c732343c", + "d0705c9e6fbaeba3", + "d1a87288683080c4", + "d2a4b8cb9ae7fe8f", + "d2b0cb5a911b8ef8", + "d2d0b2521a1f186d", + "d355c778fcfee3a9", + "d49988efd778ca9d", + "d4e1874210b91dd4", + "d50ab8fd0c085fb6", + "d699ed2b1adc6be4", + "d6d6b7c041820ab8", "d7467eab308fb4a6", - "4761b0a9b046cc66", - "0ac0120f667a8dcf", + "d75282fdda80b6fd", + "d753660b3c2bdd84", + "d837b4fde544a9f2", + "d94b13d95fd1c12a", + "dabbb27e6faa13d5", + "dadd7dd2d4247627", + "db3b58c1836b220e", + "dba4b5a166f9456a", + "dbaf9789160ca414", + "dbbf02a1cd0ad33b", + "dbf15cc4f389196f", + "dc278166b5d14512", + "dd1472476dfe4a8b", + "dd81eae9478576f3", + "de23f9f77f9b49f0", + "de9a98c311683ef3", + "df6728b8420ab3b4", + "dfe9de5d26e08949", + "e0265f0fb8e196c0", + "e0b122da55f1f002", + "e10bc3422afaec8a", + "e10d907c64d75e3a", + "e2275acc4f1c2d0d", + "e2e705cabd42e40e", + "e66a9b22df7fe32d", + "e676143c80802a59", + "e6e8add5e4db2d9d", + "e7bfc160fd3c0d5d", + "e7e366f3619b5953", + "e86db251be2c8a60", + "e886bfa3a4f4ffd4", + "e8e5f42a4e4005c7", + "e9b7e07dc5598afe", + "e9e12b044b495c6f", + "e9fc2f8b110e71a0", + "ea6d47427a2bb349", + "eac7be8886cb755b", + "ead1076787fdd7e4", + "ec84060a194fd9cc", + "ecba01acba8df2ec", + "ecfb020c517d37c1", "ed10036ddfbda912", - "00efeb17bcaafce6", - "61bff064f593ecc0", - "c3549567b1ad7bdf", - "371dca18821f7714", - "a6cdbfd0a7bef417", - "c175537338906951", - "c7f6cb518fb19151", - "6291bcc06184d125", - "06e505abb8dad1b8", - "82b7b915b7af9c17", - "725b6b95cd47b601", - "4bede5d2af4cb11f", - "600e2a1d0f05afd9", - "03281d5d8a77be56", - "27b4366a0f1fcdef", - "42799014c183e1b3", - "9c6440ecec046cc0", - "b26c9f300c14239d", - "366786e98797ebf3", - "454f266801ae2ae0", - "fdbbd2b53cb2e1e3", - "76610bf4610246c7", - "09963e79b21cde12", - "457fcec1ad31841f", - "6962629b904be54d", - "a7d6e1aeb5e009f0", - "2674228d88c4f3d6", - "2bb4685a03857b2c", - "4d776db86abecdda", - "21c3236216f620ff", - "5a12431e371f3ce9", - "2d2ce200fea72359", - "9aa4b513807ea22c", - "4526aa14337a2b0c", + "ed22d4c0f09c3e3a", + "ee1c6dd97918d74a", + "eea87713e11a55c2", + "eeb2e28f54484dc9", + "f02cf5078e4548e0", + "f041e5b526e79ef4", + "f0ce40dfe333e918", + "f17fe9ceea628f62", + "f20be2311365981c", + "f2dc875bd399a806", + "f2e00a1fee27bdd7", + "f32a1e7e2843c096", + "f3a461d6bf980196", + "f6147b5c0a147123", + "f6579adcbf5faa99", "f659401e91439524", - "dbbf02a1cd0ad33b" + "f71fca5fc684df8b", + "f73d08f8cdfe276d", + "f869956d14ea1694", + "f87b5bc03e18d010", + "f921d39906e6d93d", + "fa67fbbcfc0b9442", + "fb9ec9e846ea4e87", + "fbb507bc72376d36", + "fc373293fb7c56ac", + "fca2af05ee78ac5c", + "fdbbd2b53cb2e1e3", + "fe223a9f01625d62", + "fe445dc1c13738d6" ] \ No newline at end of file -- cgit v1.3.1 From b24d661c096b078e622495f2bbd5c1d3081e8f74 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 14 Sep 2023 10:14:24 +0200 Subject: sort values in train.csv annotations --- .../manual_annotations/train.csv | 394 ++++++++++----------- 1 file changed, 197 insertions(+), 197 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv index 3d5a0ab5..0b47364c 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv @@ -1,197 +1,197 @@ -,dgst,referenced_cert_id,source,label,comment -45,00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None -38,00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None -39,00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None -87,00b539d629e60965,CCEVS-VR-02-0025,None,None,None -79,00eb3e5c7a4537b1,BSI-DSZ-CC-0399-2007,None,None,None -80,00eb3e5c7a4537b1,BSI-DSZ-CC-0322-2005,None,None,None -10,00efeb17bcaafce6,ANSSI-CC-2009/34,report,component_used,None -37,017f3cea8275a3e1,ANSSI-CC-2017/23,None,None,None -2,01ce6f422bec823d,BSI-DSZ-CC-1040-2019,None,None,None -26,01dc77d7f8f94d97,ANSSI-CC-2017/39,None,None,None -27,01dc77d7f8f94d97,ANSSI-CC-2016/78,None,None,None -49,01e5c6d31f763f56,BSI-DSZ-CC-0917-2014,None,None,None -98,01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,report,component_used,None -46,0208e6b60a0b1a32,ANSSI-CC-2016/01,None,None,None -0,02c6fbcd90dcf05a,BSI-DSZ-CC-1107-V3-2022,None,None,None -20,02cb3c385865ad2c,BSI-DSZ-CC-1184-2022,None,None,None -21,0319b03323f257d1,BSI-DSZ-CC-1121-2019,None,None,None -22,0319b03323f257d1,BSI-DSZ-CC-1118-2020,None,None,None -47,032faef0207b715f,BSI-DSZ-CC-0837-V2-2014,None,None,None -49,037577fc2019fcfa,ANSSI-CC-2021/49,report,previous_version,None -48,04c5a5a66fb7f89a,BSI-DSZ-CC-0827-V2-2014,None,None,None -78,04cb360502da6492,BSI-DSZ-CC-0344-2005,None,None,None -95,054bd8196e015197,ANSSI-CC-2011/07,None,None,None -68,055985699aee1e09,ANSSI-CC-2010/02,report,component_used,None -58,05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,report,component_used,None -85,05d96aada7bb839c,BSI-DSZ-CC-0223-2003,None,None,None -60,06b24bc51eb69188,BSI-DSZ-CC-0813-2012,None,None,None -66,06e505abb8dad1b8,ANSSI-CC-2017/54,report,component_used,None -44,06e70fa70393d60f,BSI-DSZ-CC-0891-V2-2016,None,None,None -43,06e70fa70393d60f,ANSSI-CC-2017/07,None,None,None -33,075ddd00e07fa7ed,ANSSI-CC-2020/76,None,None,None -34,075ddd00e07fa7ed,BSI-DSZ-CC-1136-2021,None,None,None -24,07ce7e4844f62c5c,BSI-DSZ-CC-1070-2018,None,None,None -25,07ce7e4844f62c5c,ANSSI-CC-2017/73,None,None,None -64,080aa6966d757f05,ANSSI-CC-2010/02,None,None,None -63,080aa6966d757f05,ANSSI-CC-2010/05,None,None,None -17,08408cbc394e7421,BSI-DSZ-CC-0782-V4-2018,None,None,None -18,08408cbc394e7421,ANSSI-CC-2017/59,None,None,None -13,09220515a7edf3f9,BSI-DSZ-CC-1110-V2-2019,None,None,None -77,096b05114ee5e09f,BSI-DSZ-CC-0411-2007,None,None,None -76,096b05114ee5e09f,BSI-DSZ-CC-0410-2007,None,None,None -75,096b05114ee5e09f,ANSSI-CC-2008/45,None,None,None -90,096ccb6bcb3d2731,BSI-DSZ-CC-0564-2009,None,None,None -19,09b07fbbb24e1b04,BSI-DSZ-CC-0964-2015,None,None,None -72,09f539aedefa9756,ANSSI-CC-2008/05,None,None,None -70,09f539aedefa9756,ANSSI-CC-2009/11,None,None,None -71,09f539aedefa9756,ANSSI-CC-2009/41,None,None,None -40,0a6e50445a906052,BSI-DSZ-CC-0945-2017,None,None,None -41,0a6e50445a906052,BSI-DSZ-CC-0891-V2-2016,None,None,None -66,0aad0176e35b3749,BSI-DSZ-CC-0477-2007,None,None,None -22,0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None -31,0bd651fcbb86f2a9,BSI-DSZ-CC-1136-2021,None,None,None -32,0bd651fcbb86f2a9,ANSSI-CC-2020/74,None,None,None -45,0bf7a19b22163465,ANSSI-CC-2015/80,report,evaluation_reused,None -1,0bf7a19b22163465,ANSSI-CC-2016/44,report,evaluation_reused,unclear whether component used -58,0c9e215997d96f44,BSI-DSZ-CC-0555-2009,None,None,None -88,0cd406e7c5013b74,BSI-DSZ-CC-0304-2006,None,None,None -53,0d4a95a8b52b787b,BSI-DSZ-CC-0829-2012,None,None,None -30,0da9d81c9f563df5,BSI-DSZ-CC-0382-2007,None,None,None -62,0ee132dec087b772,ANSSI-CC-2011/07,None,None,None -1,0ef56cba2362e463,ANSSI-CC-2015/42,None,None,None -59,0f1b31ac93459d1f,BSI-DSZ-CC-0720-2011,None,None,None -99,0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,report,component_used,None -65,0f684159ad31f883,BSI-DSZ-CC-0555-2009,None,None,None -86,0fe0bb1054f49b7f,CCEVS-VR-04-0092,None,None,None -89,0ff82b1cc24b3105,BSI-DSZ-CC-0302-2005,None,None,None -11,10233188161519ee,ANSSI-CC-2020/26,None,None,None -10,10233188161519ee,BSI-DSZ-CC-1059-V3-2019,None,None,None -12,10233188161519ee,BSI-DSZ-CC-1059-2018,None,None,None -9,10233188161519ee,ANSSI-CC-2019/48,None,None,None -8,102707a8c4d8a1b0,BSI-DSZ-CC-1059-2018,None,None,None -7,102707a8c4d8a1b0,ANSSI-CC-2020/26,None,None,None -6,102707a8c4d8a1b0,ANSSI-CC-2019/35,None,None,None -5,102707a8c4d8a1b0,BSI-DSZ-CC-1059-V3-2019,None,None,None -15,102bbdfcd696d7a9,ANSSI-CC-2019/01,None,None,None -16,102bbdfcd696d7a9,ANSSI-CC-2018/13,None,None,None -23,107c348f4ba0a41c,BSI-DSZ-CC-1135-2020,None,None,None -67,109eb2158ca6a2f9,BSI-DSZ-CC-0410-2007,None,None,None -52,10af4a6378d0fb7a,ANSSI-CC-2014/07,None,None,None -50,10af4a6378d0fb7a,BSI-DSZ-CC-0837-V2-2014,None,None,None -51,10af4a6378d0fb7a,ANSSI-CC-2014/86,None,None,None -35,10b17081dd7cad8f,ANSSI-CC-2017/39,None,None,None -36,10b17081dd7cad8f,ANSSI-CC-2015/36,None,None,None -69,112fe9f123e277ee,ANSSI-CC-2010/02,None,None,None -68,112fe9f123e277ee,ANSSI-CC-2010/19,None,None,None -3,114e39b4da949e6d,BSI-DSZ-CC-1110-V2-2019,None,None,None -4,114e39b4da949e6d,ANSSI-CC-2020/08,None,None,None -29,1172a88e009dc248,CRP281,None,None,None -28,1172a88e009dc248,CRP257,None,None,None -73,1188aebd8681e4fe,BSI-DSZ-CC-0465-2008,None,None,None -82,122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None,None -81,122245dd15683e26,ANSSI-CC-2007/08,None,None,None -91,123c94418ddf1ad8,BSI-DSZ-CC-0367-2006,None,None,None -7,12e20466a0f08342,ANSSI-CC-2016/79,report,evaluation_reused,unclear whether component used -14,131ec924c42ef90b,ANSSI-CC-2017/61,None,None,None -42,135ca3602f0005bf,NSCIB-CC-16-99111-CR2,None,None,None -47,13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,report,component_shared,"BAC, EAC thingy" -83,13e5b88ad3becb77,BSI-DSZ-CC-0322-2005,None,None,None -84,13e5b88ad3becb77,BSI-DSZ-CC-0344-2005,None,None,None -61,1468948b4f0509ab,BSI-DSZ-CC-0813-2012,None,None,None -74,157b741a0f529a0a,ANSSI-CC-2009/37,None,None,None -56,157f811757a0b29e,BSI-DSZ-CC-0675-2011,None,None,None -57,157f811757a0b29e,BSI-DSZ-CC-0633-2010,None,None,None -55,157f811757a0b29e,BSI-DSZ-CC-0555-2009,None,None,None -54,157f811757a0b29e,BSI-DSZ-CC-0745-2012,None,None,None -92,158950ed4bc35274,ANSSI-CC-2016/33,None,None,None -93,158950ed4bc35274,ANSSI-CC-2015/80,None,None,None -94,158950ed4bc35274,ANSSI-CC-2015/36,None,None,None -42,16bdbde359584f99,ANSSI-CC-2017/61,report,component_used,None -50,17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,report,component_used,None -72,19c60923e31777e7,ANSSI-CC-2017/49,report,component_used,None -89,1b8705a0486b3be1,NSCIB-CC-0209053-CR,report,previous_version,None -46,1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,target,component_used,None -52,1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,report,component_used,None -92,27d11629261d8806,ANSSI-CC-2014/59,report,component_used,None -53,28e70f46ba4394dd,ANSSI-CC-2012/08,report,component_used,None -23,2c15dfc106ebf8d8,ANSSI-CC-2012/24,report,component_used,None -81,30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,report,component_used,None -64,30f71b100c5cebae,ANSSI-CC-2015/15,report,component_used,None -67,317713948b4473b3,BSI-DSZ-CC-0798-2012,report,component_shared,None -43,34370b67b5e675c3,ANSSI-CC-2013/47,report,component_used,None -5,3515801dee00995f,BSI-DSZ-CC-0447-2008,report,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 -97,36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,report,component_used,None -37,371dca18821f7714,BSI-DSZ-CC-0945-2017,report,component_used,None -76,3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,report,recertification,None -75,42799014c183e1b3,BSI-DSZ-CC-0429-2007,report,recertification,None -55,42799014c183e1b3,BSI-DSZ-CC-0312-2005,report,component_used,None -19,463ecd64b7506048,ANSSI-CC-2018/52,report,component_used,None -78,4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component_used,None -31,4bede5d2af4cb11f,ANSSI-CC-2019/59,report,evaluation_reused,None -40,4f5f41ecf7517e63,ANSSI-CC-2016/80,report,evaluation_reused,None -30,54273dd266fce692,ANSSI-CC-2015/15,report,component_used,None -51,5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,target,component_used,None -77,5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,target,component_used,unclear -54,61832cb4291c343f,BSI-DSZ-CC-0955-2016,report,component_used,None -56,627bfd69c2a831cf,ANSSI-CC-2014/37,report,evaluation_reused,None -4,680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,report,component_used,None -70,691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,report,component_used,None -6,6a999675c9422dfb,BSI-DSZ-CC-0353-2006,report,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 -65,6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,target,component_used,None -88,6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,report,component_used,None -86,744a7a202d909323,ANSSI-CC-2017/24,report,component_used,None -60,7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,report,recertification,None -27,845bb039719ac5d8,ANSSI-CC-2010/40,report,previous_version,None -96,84a75b6fc33b669e,ANSSI-CC-2014/08,report,evaluation_reused,None -3,8ba22f6c9651edc3,ANSSI-CC-2013/55,report,component_used,None -35,8bdb610131555c12,BSI-DSZ-CC-0645-2010,report,component_used,None -9,921e042759b30033,BSI-DSZ-CC-0978-2016,report,component_used,composition scheme “[COMP]” mentioned -59,9529eb793550093c,ANSSI-CC-2018/40,report,previous_version,None -85,97d00a9f198a6e57,ANSSI-CC-2015/66,report,component_used,None -71,a07548a64fac5794,BSI-DSZ-CC-0350-2007,report,recertification,None -33,a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,report,component_used,None -44,a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,report,previous_version,None -93,a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,report,component_used,None -41,a867d281d34d34b7,BSI-DSZ-CC-0976-2015,report,recertification,None -90,a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,report,recertification,None -17,aa096ffde94b953b,BSI-DSZ-CC-0411-2007,report,component_used,None -80,ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,report,component_used,None -2,ab3af998dff7a2ef,ANSSI-CC-2017/47,report,component_used,None -57,aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,report,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 -28,af61a31e3fd0d6f0,ANSSI-CC-2010/02,report,evaluation_reused,None -36,b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,unclear -18,b42009eb34746731,ANSSI-CC-2021/50,report,previous_version,maybe previous_version -69,b61363e51ed90c7a,ANSSI-CC-2019/12,report,component_used,None -84,bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,report,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" -79,c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,report,component_used,None -11,c175537338906951,ANSSI-CC-2016/70,report,component_shared,None -82,c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,evaluation_reused,None -95,c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,"unclear, mentions just ToE delivery" -20,c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,report,recertification,None -21,c96343baf608174d,BSI-DSZ-CC-0645-2010,report,component_used,None -74,c96343baf608174d,BSI-DSZ-CC-0750-2011,report,component_used,None -15,cacdfdafce47678c,ANSSI-CC-2014/59,target,component_used,None -91,cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,report,component_used,None -8,d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,report,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert -94,d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,report,recertification,None -29,d49988efd778ca9d,ANSSI-CC-2010/02,report,component_used,None -32,d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,report,recertification,None -16,dba4b5a166f9456a,ANSSI-CC-2014/76,report,recertification,None -87,df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,report,component_used,None -24,e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,unclear -39,e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,recertification,None -0,e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,report,component_used,the previously certified M7892 G12 component -26,e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,report,component_used,None -62,ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,report,recertification,None -63,ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,target,component_used,None -13,ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,report,component_used,None -83,ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,report,component_used,None -61,ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,report,recertification,None -25,ee1c6dd97918d74a,ANSSI-CC-2012/71,target,evaluation_reused,None -73,f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,report,component_used,None -38,f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,report,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 -34,f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,target,component_used,None -14,f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,report,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 -48,f869956d14ea1694,ANSSI-CC-2012/68,report,component_used,None -12,fb9ec9e846ea4e87,ANSSI-CC-2017/47,report,component_used,None +dgst,referenced_cert_id,source,label,comment +00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None +00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None +00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None +00b539d629e60965,CCEVS-VR-02-0025,None,None,None +00eb3e5c7a4537b1,BSI-DSZ-CC-0322-2005,None,None,None +00eb3e5c7a4537b1,BSI-DSZ-CC-0399-2007,None,None,None +00efeb17bcaafce6,ANSSI-CC-2009/34,report,component_used,None +017f3cea8275a3e1,ANSSI-CC-2017/23,None,None,None +01ce6f422bec823d,BSI-DSZ-CC-1040-2019,None,None,None +01dc77d7f8f94d97,ANSSI-CC-2016/78,None,None,None +01dc77d7f8f94d97,ANSSI-CC-2017/39,None,None,None +01e5c6d31f763f56,BSI-DSZ-CC-0917-2014,None,None,None +01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,report,component_used,None +0208e6b60a0b1a32,ANSSI-CC-2016/01,None,None,None +02c6fbcd90dcf05a,BSI-DSZ-CC-1107-V3-2022,None,None,None +02cb3c385865ad2c,BSI-DSZ-CC-1184-2022,None,None,None +0319b03323f257d1,BSI-DSZ-CC-1118-2020,None,None,None +0319b03323f257d1,BSI-DSZ-CC-1121-2019,None,None,None +032faef0207b715f,BSI-DSZ-CC-0837-V2-2014,None,None,None +037577fc2019fcfa,ANSSI-CC-2021/49,report,previous_version,None +04c5a5a66fb7f89a,BSI-DSZ-CC-0827-V2-2014,None,None,None +04cb360502da6492,BSI-DSZ-CC-0344-2005,None,None,None +054bd8196e015197,ANSSI-CC-2011/07,None,None,None +055985699aee1e09,ANSSI-CC-2010/02,report,component_used,None +05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,report,component_used,None +05d96aada7bb839c,BSI-DSZ-CC-0223-2003,None,None,None +06b24bc51eb69188,BSI-DSZ-CC-0813-2012,None,None,None +06e505abb8dad1b8,ANSSI-CC-2017/54,report,component_used,None +06e70fa70393d60f,ANSSI-CC-2017/07,None,None,None +06e70fa70393d60f,BSI-DSZ-CC-0891-V2-2016,None,None,None +075ddd00e07fa7ed,ANSSI-CC-2020/76,None,None,None +075ddd00e07fa7ed,BSI-DSZ-CC-1136-2021,None,None,None +07ce7e4844f62c5c,ANSSI-CC-2017/73,None,None,None +07ce7e4844f62c5c,BSI-DSZ-CC-1070-2018,None,None,None +080aa6966d757f05,ANSSI-CC-2010/02,None,None,None +080aa6966d757f05,ANSSI-CC-2010/05,None,None,None +08408cbc394e7421,ANSSI-CC-2017/59,None,None,None +08408cbc394e7421,BSI-DSZ-CC-0782-V4-2018,None,None,None +09220515a7edf3f9,BSI-DSZ-CC-1110-V2-2019,None,None,None +096b05114ee5e09f,ANSSI-CC-2008/45,None,None,None +096b05114ee5e09f,BSI-DSZ-CC-0410-2007,None,None,None +096b05114ee5e09f,BSI-DSZ-CC-0411-2007,None,None,None +096ccb6bcb3d2731,BSI-DSZ-CC-0564-2009,None,None,None +09b07fbbb24e1b04,BSI-DSZ-CC-0964-2015,None,None,None +09f539aedefa9756,ANSSI-CC-2008/05,None,None,None +09f539aedefa9756,ANSSI-CC-2009/11,None,None,None +09f539aedefa9756,ANSSI-CC-2009/41,None,None,None +0a6e50445a906052,BSI-DSZ-CC-0891-V2-2016,None,None,None +0a6e50445a906052,BSI-DSZ-CC-0945-2017,None,None,None +0aad0176e35b3749,BSI-DSZ-CC-0477-2007,None,None,None +0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None +0bd651fcbb86f2a9,ANSSI-CC-2020/74,None,None,None +0bd651fcbb86f2a9,BSI-DSZ-CC-1136-2021,None,None,None +0bf7a19b22163465,ANSSI-CC-2015/80,report,evaluation_reused,None +0bf7a19b22163465,ANSSI-CC-2016/44,report,evaluation_reused,unclear whether component used +0c9e215997d96f44,BSI-DSZ-CC-0555-2009,None,None,None +0cd406e7c5013b74,BSI-DSZ-CC-0304-2006,None,None,None +0d4a95a8b52b787b,BSI-DSZ-CC-0829-2012,None,None,None +0da9d81c9f563df5,BSI-DSZ-CC-0382-2007,None,None,None +0ee132dec087b772,ANSSI-CC-2011/07,None,None,None +0ef56cba2362e463,ANSSI-CC-2015/42,None,None,None +0f1b31ac93459d1f,BSI-DSZ-CC-0720-2011,None,None,None +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,report,component_used,None +0f684159ad31f883,BSI-DSZ-CC-0555-2009,None,None,None +0fe0bb1054f49b7f,CCEVS-VR-04-0092,None,None,None +0ff82b1cc24b3105,BSI-DSZ-CC-0302-2005,None,None,None +10233188161519ee,ANSSI-CC-2019/48,None,None,None +10233188161519ee,ANSSI-CC-2020/26,None,None,None +10233188161519ee,BSI-DSZ-CC-1059-2018,None,None,None +10233188161519ee,BSI-DSZ-CC-1059-V3-2019,None,None,None +102707a8c4d8a1b0,ANSSI-CC-2019/35,None,None,None +102707a8c4d8a1b0,ANSSI-CC-2020/26,None,None,None +102707a8c4d8a1b0,BSI-DSZ-CC-1059-2018,None,None,None +102707a8c4d8a1b0,BSI-DSZ-CC-1059-V3-2019,None,None,None +102bbdfcd696d7a9,ANSSI-CC-2018/13,None,None,None +102bbdfcd696d7a9,ANSSI-CC-2019/01,None,None,None +107c348f4ba0a41c,BSI-DSZ-CC-1135-2020,None,None,None +109eb2158ca6a2f9,BSI-DSZ-CC-0410-2007,None,None,None +10af4a6378d0fb7a,ANSSI-CC-2014/07,None,None,None +10af4a6378d0fb7a,ANSSI-CC-2014/86,None,None,None +10af4a6378d0fb7a,BSI-DSZ-CC-0837-V2-2014,None,None,None +10b17081dd7cad8f,ANSSI-CC-2015/36,None,None,None +10b17081dd7cad8f,ANSSI-CC-2017/39,None,None,None +112fe9f123e277ee,ANSSI-CC-2010/02,None,None,None +112fe9f123e277ee,ANSSI-CC-2010/19,None,None,None +114e39b4da949e6d,ANSSI-CC-2020/08,None,None,None +114e39b4da949e6d,BSI-DSZ-CC-1110-V2-2019,None,None,None +1172a88e009dc248,CRP257,None,None,None +1172a88e009dc248,CRP281,None,None,None +1188aebd8681e4fe,BSI-DSZ-CC-0465-2008,None,None,None +122245dd15683e26,ANSSI-CC-2007/08,None,None,None +122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None,None +123c94418ddf1ad8,BSI-DSZ-CC-0367-2006,None,None,None +12e20466a0f08342,ANSSI-CC-2016/79,report,evaluation_reused,unclear whether component used +131ec924c42ef90b,ANSSI-CC-2017/61,None,None,None +135ca3602f0005bf,NSCIB-CC-16-99111-CR2,None,None,None +13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,report,component_shared,"BAC, EAC thingy" +13e5b88ad3becb77,BSI-DSZ-CC-0322-2005,None,None,None +13e5b88ad3becb77,BSI-DSZ-CC-0344-2005,None,None,None +1468948b4f0509ab,BSI-DSZ-CC-0813-2012,None,None,None +157b741a0f529a0a,ANSSI-CC-2009/37,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0555-2009,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0633-2010,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0675-2011,None,None,None +157f811757a0b29e,BSI-DSZ-CC-0745-2012,None,None,None +158950ed4bc35274,ANSSI-CC-2015/36,None,None,None +158950ed4bc35274,ANSSI-CC-2015/80,None,None,None +158950ed4bc35274,ANSSI-CC-2016/33,None,None,None +16bdbde359584f99,ANSSI-CC-2017/61,report,component_used,None +17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,report,component_used,None +19c60923e31777e7,ANSSI-CC-2017/49,report,component_used,None +1b8705a0486b3be1,NSCIB-CC-0209053-CR,report,previous_version,None +1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,target,component_used,None +1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,report,component_used,None +27d11629261d8806,ANSSI-CC-2014/59,report,component_used,None +28e70f46ba4394dd,ANSSI-CC-2012/08,report,component_used,None +2c15dfc106ebf8d8,ANSSI-CC-2012/24,report,component_used,None +30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,report,component_used,None +30f71b100c5cebae,ANSSI-CC-2015/15,report,component_used,None +317713948b4473b3,BSI-DSZ-CC-0798-2012,report,component_shared,None +34370b67b5e675c3,ANSSI-CC-2013/47,report,component_used,None +3515801dee00995f,BSI-DSZ-CC-0447-2008,report,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 +36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,report,component_used,None +371dca18821f7714,BSI-DSZ-CC-0945-2017,report,component_used,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,report,recertification,None +42799014c183e1b3,BSI-DSZ-CC-0312-2005,report,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,report,recertification,None +463ecd64b7506048,ANSSI-CC-2018/52,report,component_used,None +4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component_used,None +4bede5d2af4cb11f,ANSSI-CC-2019/59,report,evaluation_reused,None +4f5f41ecf7517e63,ANSSI-CC-2016/80,report,evaluation_reused,None +54273dd266fce692,ANSSI-CC-2015/15,report,component_used,None +5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,target,component_used,None +5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,target,component_used,unclear +61832cb4291c343f,BSI-DSZ-CC-0955-2016,report,component_used,None +627bfd69c2a831cf,ANSSI-CC-2014/37,report,evaluation_reused,None +680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,report,component_used,None +691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,report,component_used,None +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,report,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,target,component_used,None +6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,report,component_used,None +744a7a202d909323,ANSSI-CC-2017/24,report,component_used,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,report,recertification,None +845bb039719ac5d8,ANSSI-CC-2010/40,report,previous_version,None +84a75b6fc33b669e,ANSSI-CC-2014/08,report,evaluation_reused,None +8ba22f6c9651edc3,ANSSI-CC-2013/55,report,component_used,None +8bdb610131555c12,BSI-DSZ-CC-0645-2010,report,component_used,None +921e042759b30033,BSI-DSZ-CC-0978-2016,report,component_used,composition scheme “[COMP]” mentioned +9529eb793550093c,ANSSI-CC-2018/40,report,previous_version,None +97d00a9f198a6e57,ANSSI-CC-2015/66,report,component_used,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,report,recertification,None +a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,report,component_used,None +a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,report,previous_version,None +a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,report,component_used,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,report,recertification,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,report,recertification,None +aa096ffde94b953b,BSI-DSZ-CC-0411-2007,report,component_used,None +ab3af998dff7a2ef,ANSSI-CC-2017/47,report,component_used,None +ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,report,component_used,None +aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,report,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 +af61a31e3fd0d6f0,ANSSI-CC-2010/02,report,evaluation_reused,None +b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,unclear +b42009eb34746731,ANSSI-CC-2021/50,report,previous_version,maybe previous_version +b61363e51ed90c7a,ANSSI-CC-2019/12,report,component_used,None +bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,report,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" +c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,report,component_used,None +c175537338906951,ANSSI-CC-2016/70,report,component_shared,None +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,evaluation_reused,None +c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,"unclear, mentions just ToE delivery" +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,report,recertification,None +c96343baf608174d,BSI-DSZ-CC-0645-2010,report,component_used,None +c96343baf608174d,BSI-DSZ-CC-0750-2011,report,component_used,None +cacdfdafce47678c,ANSSI-CC-2014/59,target,component_used,None +cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,report,component_used,None +d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,report,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,report,recertification,None +d49988efd778ca9d,ANSSI-CC-2010/02,report,component_used,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,report,recertification,None +dba4b5a166f9456a,ANSSI-CC-2014/76,report,recertification,None +df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,report,component_used,None +e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,unclear +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,recertification,None +e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,report,component_used,the previously certified M7892 G12 component +e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,report,component_used,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,report,recertification,None +ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,target,component_used,None +ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,report,component_used,None +ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,report,component_used,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,report,recertification,None +ee1c6dd97918d74a,ANSSI-CC-2012/71,target,evaluation_reused,None +f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,report,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,report,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 +f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,target,component_used,None +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,report,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +f869956d14ea1694,ANSSI-CC-2012/68,report,component_used,None +fb9ec9e846ea4e87,ANSSI-CC-2017/47,report,component_used,None -- cgit v1.3.1 From 677bd62f02da6697ac9386d94450bf118e117e95 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 14 Sep 2023 11:00:17 +0200 Subject: delete one test annotation for which we dont have cert --- src/sec_certs/data/reference_annotations/manual_annotations/test.csv | 1 - 1 file changed, 1 deletion(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/test.csv b/src/sec_certs/data/reference_annotations/manual_annotations/test.csv index fd43357f..03669faa 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/test.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/test.csv @@ -749,7 +749,6 @@ a67919286833a7df,BSI-DSZ-CC-0957-V2-2016,None,None,None a6c1dd30884a197c,ANSSI-CC-2010/08,None,None,None a76054a284f0ce61,ANSSI-CC-2014/86,None,None,None a76054a284f0ce61,BSI-DSZ-CC-0837-V2-2014,None,None,None -a781ac2579798523,BSI-DSZ-CC-1059-V4-2021,None,None,None a851c3b4377f4923,ANSSI-CC-2011/07,None,None,None a9004216663480de,BSI-DSZ-CC-0782-V2-2015,None,None,None a925d8f1adc5cfe2,BSI-DSZ-CC-0891-V3-2018,None,None,None -- cgit v1.3.1 From 6461732e3fc844be4219bc02eb775d90c42f7b18 Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Fri, 15 Sep 2023 13:35:14 +0200 Subject: delete nonexisting cert from test split --- src/sec_certs/data/reference_annotations/split/test.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/split/test.json b/src/sec_certs/data/reference_annotations/split/test.json index 2e761490..9d7195d0 100644 --- a/src/sec_certs/data/reference_annotations/split/test.json +++ b/src/sec_certs/data/reference_annotations/split/test.json @@ -436,7 +436,6 @@ "a67919286833a7df", "a6c1dd30884a197c", "a76054a284f0ce61", - "a781ac2579798523", "a851c3b4377f4923", "a9004216663480de", "a925d8f1adc5cfe2", @@ -711,4 +710,4 @@ "ffd213e55eee8cc8", "ffe4328dcdc6b538", "fff289a66cb9ce4b" -] \ No newline at end of file +] -- cgit v1.3.1 From 639ba7ba1290fc7bbd218d8a0006ac462ac7b6b7 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Sun, 17 Sep 2023 14:10:22 +0200 Subject: WiP: introduce actually extracted cert_id_keywords instead of canonical --- .../manual_annotations/test.csv | 2 +- .../manual_annotations/train.csv | 2 +- .../manual_annotations/valid.csv | 2 +- .../model/references/segment_extractor.py | 79 +++++++++++++++++----- src/sec_certs/sample/cc_certificate_id.py | 7 +- 5 files changed, 72 insertions(+), 20 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/test.csv b/src/sec_certs/data/reference_annotations/manual_annotations/test.csv index 03669faa..ad952755 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/test.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/test.csv @@ -1,4 +1,4 @@ -dgst,referenced_cert_id,source,label,comment +dgst,canonical_reference_keyword,source,label,comment 00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None 00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None 00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv index 0b47364c..2cab70dd 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv @@ -1,4 +1,4 @@ -dgst,referenced_cert_id,source,label,comment +dgst,canonical_reference_keyword,source,label,comment 00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None 00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None 00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv b/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv index ffa897e7..e1896fe3 100644 --- a/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv +++ b/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv @@ -1,4 +1,4 @@ -dgst,referenced_cert_id,source,label,comment +dgst,canonical_reference_keyword,source,label,comment a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,report,component_used,None 657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,report,recertification,None 0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,report,component_used,None diff --git a/src/sec_certs/model/references/segment_extractor.py b/src/sec_certs/model/references/segment_extractor.py index bc3869d2..95a8ec2a 100644 --- a/src/sec_certs/model/references/segment_extractor.py +++ b/src/sec_certs/model/references/segment_extractor.py @@ -5,7 +5,7 @@ import json import re from dataclasses import dataclass from pathlib import Path -from typing import Iterable, Literal +from typing import Any, Iterable, Literal import langdetect import pandas as pd @@ -13,11 +13,23 @@ import spacy from importlib_resources import files from sec_certs.sample.cc import CCCertificate +from sec_certs.sample.cc_certificate_id import CertificateId from sec_certs.utils import parallel_processing nlp = spacy.load("en_core_web_sm") +def swap_and_filter_dict(dct: dict[str, Any], filter_to_keys: set[str]): + new_dct: dict[str, set[str]] = {} + for key, val in dct.items(): + if val in new_dct: + new_dct[val].add(key) + else: + new_dct[val] = {key} + + return {key: val for key, val in new_dct.items() if key in filter_to_keys} + + def references_to_label_studio(df: pd.DataFrame, filepath: Path) -> None: """ Prepares a DataFrame obtained from ReferenceSegmentExtractor to be used in Label Studio for manual annotation. @@ -41,7 +53,9 @@ def fill_reference_segments(record: ReferenceRecord) -> ReferenceRecord: with record.processed_data_source_path.open("r") as handle: data = handle.read() - sentences_with_hits = [sent.text for sent in nlp(data).sents if record.referenced_cert_id in sent.text] + sentences_with_hits = [ + sent.text for sent in nlp(data).sents if any(x in sent.text for x in record.actual_reference_keywords) + ] if not sentences_with_hits: record.segments = None return record @@ -75,7 +89,7 @@ def preprocess_data_source(record: ReferenceRecord) -> ReferenceRecord: with record.raw_data_source_path.open("r") as handle: data = handle.read() - processed_data = preprocess_txt_func(data, record.referenced_cert_id) + processed_data = preprocess_txt_func(data, record.canonical_reference_keyword) with record.processed_data_source_path.open("w") as handle: handle.write(processed_data) @@ -137,12 +151,19 @@ class ReferenceRecord: certificate_dgst: str raw_data_source_path: Path processed_data_source_path: Path - referenced_cert_id: str + canonical_reference_keyword: str + actual_reference_keywords: set[str] source: str segments: set[str] | None = None - def to_pandas_tuple(self) -> tuple[str, str, str, set[str] | None]: - return self.certificate_dgst, self.referenced_cert_id, self.source, self.segments + def to_pandas_tuple(self) -> tuple[str, str, set[str], str, set[str] | None]: + return ( + self.certificate_dgst, + self.canonical_reference_keyword, + self.actual_reference_keywords, + self.source, + self.segments, + ) class ReferenceSegmentExtractor: @@ -178,18 +199,32 @@ class ReferenceSegmentExtractor: def _build_records(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> list[ReferenceRecord]: def get_cert_records(cert: CCCertificate, source: Literal["target", "report"]) -> list[ReferenceRecord]: - ref_var = {"target": "st_references", "report": "report_references"} + canonical_ref_var = {"target": "st_references", "report": "report_references"} + actual_ref_var = {"target": "st_keywords", "report": "report_keywords"} raw_source_var = {"target": "st_txt_path", "report": "report_txt_path"} - references = getattr(cert.heuristics, ref_var[source]).directly_referencing + canonical_references = getattr(cert.heuristics, canonical_ref_var[source]).directly_referencing + actual_references = getattr(cert.pdf_data, actual_ref_var[source])["cc_cert_id"] + actual_references = { + inner_key: CertificateId(outer_key, inner_key).canonical + for outer_key, val in actual_references.items() + for inner_key in val + } + actual_references = swap_and_filter_dict(actual_references, canonical_references) + raw_source_dir = getattr(cert.state, raw_source_var[source]).parent processed_source_dir = raw_source_dir.parent / "txt_processed" return [ ReferenceRecord( - cert.dgst, raw_source_dir / f"{cert.dgst}.txt", processed_source_dir / f"{cert.dgst}.txt", x, source + cert.dgst, + raw_source_dir / f"{cert.dgst}.txt", + processed_source_dir / f"{cert.dgst}.txt", + key, + val, + source, ) - for x in references + for key, val in actual_references.items() ] (certs[0].state.report_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True) @@ -216,7 +251,7 @@ class ReferenceSegmentExtractor: print(f"I now have {len(results)} in {source} mode") return pd.DataFrame.from_records( [x.to_pandas_tuple() for x in results], - columns=["dgst", "referenced_cert_id", "source", "segments"], + columns=["dgst", "canonical_reference_keyword", "actual_reference_keywords", "source", "segments"], ) @staticmethod @@ -239,7 +274,7 @@ class ReferenceSegmentExtractor: @staticmethod def _get_annotations_dict() -> dict[tuple[str, str], str]: """ - Returns dictionary mapping tuples `(dgst, referenced_cert_id) -> label` + Returns dictionary mapping tuples `(dgst, canonical_reference_keyword) -> label` """ def load_single_df(pth: Path, split_name: str) -> pd.DataFrame: @@ -260,7 +295,9 @@ class ReferenceSegmentExtractor: ) return ( - df_annot[["dgst", "referenced_cert_id", "label"]].set_index(["dgst", "referenced_cert_id"]).label.to_dict() + df_annot[["dgst", "canonical_reference_keyword", "label"]] + .set_index(["dgst", "canonical_reference_keyword"]) + .label.to_dict() ) @staticmethod @@ -276,13 +313,23 @@ class ReferenceSegmentExtractor: .explode("segments") .assign(lang=lambda df_: df_.segments.map(langdetect.detect)) .loc[lambda df_: df_.lang.isin({"en", "fr", "de"})] - .groupby(["dgst", "referenced_cert_id", "source"], as_index=False, dropna=False) + .groupby( + ["dgst", "canonical_reference_keyword", "actual_reference_keywords", "source"], + as_index=False, + dropna=False, + ) .agg({"segments": list, "lang": list}) .assign( split=lambda df_: df_.dgst.map(split_dct), - label=lambda df_: [annotations_dict.get(x) for x in zip(df_["dgst"], df_["referenced_cert_id"])], + label=lambda df_: [ + annotations_dict.get(x) for x in zip(df_["dgst"], df_["canonical_reference_keyword"]) + ], ) .loc[lambda df_: df_["split"] != "test"] - .groupby(["dgst", "referenced_cert_id", "label", "split"], as_index=False, dropna=False) + .groupby( + ["dgst", "canonical_reference_keyword", "actual_reference_keywords", "label", "split"], + as_index=False, + dropna=False, + ) .agg({"segments": sum, "lang": sum}) ) diff --git a/src/sec_certs/sample/cc_certificate_id.py b/src/sec_certs/sample/cc_certificate_id.py index 3e28dfc8..d0ebdb81 100644 --- a/src/sec_certs/sample/cc_certificate_id.py +++ b/src/sec_certs/sample/cc_certificate_id.py @@ -14,6 +14,11 @@ class CertificateId: raw: str def _canonical_fr(self) -> str: + def pad_last_segment_with_zero(id_str: str) -> str: + prefix, num = id_str.split("/") + num = num.zfill(2) + return f"{prefix}/{num}" + new_cert_id = self.clean rules = [ "(?:Rapport de certification|Certification Report) ([0-9]+[/-_][0-9]+(?:[vV][1-9])?(?:[_/-][MSR][0-9]+)?)", @@ -22,7 +27,7 @@ class CertificateId: ] for rule in rules: if match := re.match(rule, new_cert_id): - return "ANSSI-CC-" + match.group(1).replace("_", "/").replace("V", "v") + return pad_last_segment_with_zero("ANSSI-CC-" + match.group(1).replace("_", "/").replace("V", "v")) return new_cert_id -- cgit v1.3.1 From 597f25e13f9b56508aaf67aef4d518285ca6e676 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Thu, 19 Oct 2023 14:22:48 +0200 Subject: finalize annotation labels --- .../cc/reference_annotations/prediction.ipynb | 24 +- .../cc/reference_annotations/tf_idf_baseline.ipynb | 10 +- .../data/reference_annotations/adam/test.csv | 201 ++++ .../data/reference_annotations/adam/train.csv | 101 ++ .../data/reference_annotations/adam/valid.csv | 108 ++ .../data/reference_annotations/conflicts/test.csv | 47 + .../data/reference_annotations/conflicts/train.csv | 10 + .../data/reference_annotations/conflicts/valid.csv | 17 + .../data/reference_annotations/final/test.csv | 201 ++++ .../data/reference_annotations/final/train.csv | 101 ++ .../data/reference_annotations/final/valid.csv | 108 ++ .../data/reference_annotations/jano/test.csv | 201 ++++ .../data/reference_annotations/jano/train.csv | 101 ++ .../data/reference_annotations/jano/valid.csv | 108 ++ .../manual_annotations/test.csv | 1171 -------------------- .../manual_annotations/train.csv | 197 ---- .../manual_annotations/valid.csv | 108 -- src/sec_certs/data/reference_annotations/readme.md | 32 +- .../model/references/segment_extractor.py | 2 +- 19 files changed, 1338 insertions(+), 1510 deletions(-) create mode 100644 src/sec_certs/data/reference_annotations/adam/test.csv create mode 100644 src/sec_certs/data/reference_annotations/adam/train.csv create mode 100644 src/sec_certs/data/reference_annotations/adam/valid.csv create mode 100644 src/sec_certs/data/reference_annotations/conflicts/test.csv create mode 100644 src/sec_certs/data/reference_annotations/conflicts/train.csv create mode 100644 src/sec_certs/data/reference_annotations/conflicts/valid.csv create mode 100644 src/sec_certs/data/reference_annotations/final/test.csv create mode 100644 src/sec_certs/data/reference_annotations/final/train.csv create mode 100644 src/sec_certs/data/reference_annotations/final/valid.csv create mode 100644 src/sec_certs/data/reference_annotations/jano/test.csv create mode 100644 src/sec_certs/data/reference_annotations/jano/train.csv create mode 100644 src/sec_certs/data/reference_annotations/jano/valid.csv delete mode 100644 src/sec_certs/data/reference_annotations/manual_annotations/test.csv delete mode 100644 src/sec_certs/data/reference_annotations/manual_annotations/train.csv delete mode 100644 src/sec_certs/data/reference_annotations/manual_annotations/valid.csv (limited to 'src/sec_certs/data/reference_annotations') diff --git a/notebooks/cc/reference_annotations/prediction.ipynb b/notebooks/cc/reference_annotations/prediction.ipynb index 3b743407..628f3204 100644 --- a/notebooks/cc/reference_annotations/prediction.ipynb +++ b/notebooks/cc/reference_annotations/prediction.ipynb @@ -36,14 +36,14 @@ "\n", "REPO_ROOT = Path(\".\").resolve()\n", "DATASET_PATH = REPO_ROOT / \"dataset/cc_final_run_may_23/dataset.json\"\n", - "ANNOTATIONS_PATH = REPO_ROOT / \"src/sec_certs/data/reference_annotations/manual_annotations/\"\n", + "ANNOTATIONS_PATH = REPO_ROOT / \"src/sec_certs/data/reference_annotations/final/\"\n", "\n", "def replace_all(text: str, to_replce: set[str]) -> str:\n", " for i in to_replce:\n", " text = text.replace(i, \"\")\n", " return text\n", "\n", - "print(f\"GPU available: {torch.cuda.is_available()}\")" + "print(f\"GPU available: {torch.cuda.is_available()}\")\n" ] }, { @@ -80,7 +80,7 @@ "all_annotations[\"referenced_cert_name_stripped_version\"] = all_annotations.apply(lambda x: replace_all(x[\"referenced_cert_name\"], x[\"referenced_cert_versions\"]), axis=1)\n", "all_annotations[\"name_similarity\"] = all_annotations.apply(lambda x: fuzz.token_set_ratio(x[\"cert_name\"], x[\"referenced_cert_name\"]), axis=1)\n", "all_annotations[\"name_similarity_stripped_version\"] = all_annotations.apply(lambda x: fuzz.token_set_ratio(x[\"cert_name_stripped_version\"], x[\"referenced_cert_name_stripped_version\"]), axis=1)\n", - "all_annotations[\"name_len_diff\"] = all_annotations.apply(lambda x: abs(len(x[\"cert_name_stripped_version\"]) - len(x[\"referenced_cert_name_stripped_version\"])), axis=1)" + "all_annotations[\"name_len_diff\"] = all_annotations.apply(lambda x: abs(len(x[\"cert_name_stripped_version\"]) - len(x[\"referenced_cert_name_stripped_version\"])), axis=1)\n" ] }, { @@ -102,7 +102,7 @@ "\n", "# Simplified binary labels\n", "# label_mapping = {\"COMPONENT_USED\": \"COMPONENT_SHARED\", \"RECERTIFICATION\": \"PREVIOUS_VERSION\"}\n", - "# df.label = df.label.map(lambda x: label_mapping[x] if x in label_mapping else x)" + "# df.label = df.label.map(lambda x: label_mapping[x] if x in label_mapping else x)\n" ] }, { @@ -122,7 +122,7 @@ " segment = segment.replace(referenced_cert_id, \"the referenced product\")\n", " return segment\n", "\n", - "df.segments = df.apply(lambda row: [process_segment(x, row.referenced_cert_id) for x in row.segments], axis=1)" + "df.segments = df.apply(lambda row: [process_segment(x, row.referenced_cert_id) for x in row.segments], axis=1)\n" ] }, { @@ -141,7 +141,7 @@ "dummy_clf = DummyClassifier()\n", "dummy_clf.fit(df.loc[df.split == \"train\", [\"segments\"]], df.loc[df.split == \"train\"].label)\n", "y_pred_dummy = dummy_clf.predict(df.loc[df.split == \"valid\", [\"segments\"]])\n", - "print(classification_report(df.loc[df.split == \"valid\"].label, y_pred_dummy, zero_division=0))" + "print(classification_report(df.loc[df.split == \"valid\"].label, y_pred_dummy, zero_division=0))\n" ] }, { @@ -171,7 +171,7 @@ "\n", "# Serialize errors into file\n", "df_predicted.y_proba = df_predicted.y_proba.map(lambda x: {y: z for y, z in zip(trainer.label_mapping.values(), x)})\n", - "df_predicted.loc[~df_predicted.correct].to_json(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/annotator_errors.json\", orient=\"records\", indent=4)" + "df_predicted.loc[~df_predicted.correct].to_json(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/annotator_errors.json\", orient=\"records\", indent=4)\n" ] }, { @@ -199,10 +199,10 @@ "def objective(trial):\n", " trainer = define_trainer(trial, df)\n", " trainer.train()\n", - " \n", + "\n", " annotator = trainer.clf\n", " df_predicted = annotator.predict_df(df)\n", - " return f1_score(df_predicted.loc[df_predicted.split == \"valid\", [\"y_pred\"]], df_predicted.loc[df_predicted.split == \"valid\", [\"label\"]], zero_division=\"warn\", average=\"weighted\")" + " return f1_score(df_predicted.loc[df_predicted.split == \"valid\", [\"y_pred\"]], df_predicted.loc[df_predicted.split == \"valid\", [\"label\"]], zero_division=\"warn\", average=\"weighted\")\n" ] }, { @@ -216,7 +216,7 @@ "\n", "best_trial = study.best_trial\n", "print(\"Best Trial:\", best_trial.params)\n", - "print(\"Best Trial Value:\", best_trial.value)" + "print(\"Best Trial Value:\", best_trial.value)\n" ] }, { @@ -232,7 +232,7 @@ "ax.figure.savefig(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/cc_refs_hyperparam_search/param_importances.pdf\", bbox_inches=\"tight\")\n", "\n", "ax = optuna.visualization.matplotlib.plot_timeline(study)\n", - "ax.figure.savefig(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/cc_refs_hyperparam_search/timeline.pdf\", bbox_inches=\"tight\")" + "ax.figure.savefig(\"/var/tmp/xjanovsk/certs/sec-certs/dataset/cc_refs_hyperparam_search/timeline.pdf\", bbox_inches=\"tight\")\n" ] }, { @@ -358,7 +358,7 @@ } ], "source": [ - "study.trials_dataframe()" + "study.trials_dataframe()\n" ] } ], diff --git a/notebooks/cc/reference_annotations/tf_idf_baseline.ipynb b/notebooks/cc/reference_annotations/tf_idf_baseline.ipynb index c0396003..d7668a8a 100644 --- a/notebooks/cc/reference_annotations/tf_idf_baseline.ipynb +++ b/notebooks/cc/reference_annotations/tf_idf_baseline.ipynb @@ -32,14 +32,14 @@ "\n", "REPO_ROOT = Path(\".\").resolve()\n", "DATASET_PATH = REPO_ROOT / \"dataset/cc_final_run_may_23/dataset.json\"\n", - "ANNOTATIONS_PATH = REPO_ROOT / \"src/sec_certs/data/reference_annotations/manual_annotations/\"\n", + "ANNOTATIONS_PATH = REPO_ROOT / \"src/sec_certs/data/reference_annotations/final/\"\n", "\n", "def replace_all(text: str, to_replce: set[str]) -> str:\n", " for i in to_replce:\n", " text = text.replace(i, \"\")\n", " return text\n", "\n", - "print(f\"GPU available: {torch.cuda.is_available()}\")" + "print(f\"GPU available: {torch.cuda.is_available()}\")\n" ] }, { @@ -73,7 +73,7 @@ "all_annotations[\"cert_name_stripped_version\"] = all_annotations.apply(lambda x: replace_all(x[\"cert_name\"], x[\"cert_versions\"]), axis=1)\n", "all_annotations[\"referenced_cert_name_stripped_version\"] = all_annotations.apply(lambda x: replace_all(x[\"referenced_cert_name\"], x[\"referenced_cert_versions\"]), axis=1)\n", "all_annotations[\"name_similarity\"] = all_annotations.apply(lambda x: fuzz.token_set_ratio(x[\"cert_name\"], x[\"referenced_cert_name\"]), axis=1)\n", - "all_annotations[\"name_similarity_stripped_version\"] = all_annotations.apply(lambda x: fuzz.token_set_ratio(x[\"cert_name_stripped_version\"], x[\"referenced_cert_name_stripped_version\"]), axis=1)" + "all_annotations[\"name_similarity_stripped_version\"] = all_annotations.apply(lambda x: fuzz.token_set_ratio(x[\"cert_name_stripped_version\"], x[\"referenced_cert_name_stripped_version\"]), axis=1)\n" ] }, { @@ -90,7 +90,7 @@ "outputs": [], "source": [ "df = ReferenceSegmentExtractor()(dset.certs.values())\n", - "df = df.loc[df.label.notnull()].copy()" + "df = df.loc[df.label.notnull()].copy()\n" ] }, { @@ -147,7 +147,7 @@ "# Finally print the classification report on the aggregated results\n", "print(classification_report(df_final.loc[df_final.split == \"valid\"].label, df_final.loc[df_final.split == \"valid\"].y_pred, zero_division=0))\n", "\n", - "ConfusionMatrixDisplay.from_predictions(df_final.loc[df_final.split == \"valid\"].label, df_final.loc[df_final.split == \"valid\"].y_pred, xticks_rotation=90)" + "ConfusionMatrixDisplay.from_predictions(df_final.loc[df_final.split == \"valid\"].label, df_final.loc[df_final.split == \"valid\"].y_pred, xticks_rotation=90)\n" ] } ], diff --git a/src/sec_certs/data/reference_annotations/adam/test.csv b/src/sec_certs/data/reference_annotations/adam/test.csv new file mode 100644 index 00000000..19f4d790 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/adam/test.csv @@ -0,0 +1,201 @@ +dgst,canonical_reference_keyword,label,comment +15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,recertification,None +15b121492722bffb,BSI-DSZ-CC-0754-2012,recertification,None +16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,recertification,None +16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None +16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None +173097995b7a7f12,ANSSI-CC-2012/49,component_shared,None +173097995b7a7f12,ANSSI-CC-2013/28,evaluation_reused,None +173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None +173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None +177dee4c1051e612,BSI-DSZ-CC-0854-2013,recertification,None +17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None +183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None +183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None +18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,component_used,None +18b7e5bdf459ca13,NSCIB-CC-07-09219,component_used,None +195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None +195366bfba7213a5,BSI-DSZ-CC-0322-2005,component_used,None +195366bfba7213a5,BSI-DSZ-CC-0399-2007,recertification,None +1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None +19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None +1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,recertification,None +1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None +1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None +1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None +1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None +1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,recertification,None +1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,None +1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,evaluation_reused,None +1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,recertification,None +1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None +1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,recertification,None +2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,recertification,None +21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None +21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None +22ede463dbf1a105,ANSSI-CC-2015/36,evaluation_reused,None +22ede463dbf1a105,ANSSI-CC-2015/80,evaluation_reused,None +22ede463dbf1a105,ANSSI-CC-2016/43,evaluation_reused,None +22ede463dbf1a105,ANSSI-CC-2016/44,evaluation_reused,None +23a85a7f9c07412f,ANSSI-CC-2014/46,evaluation_reused,None +23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,None +24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None +24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None +259df98e7476a843,BSI-DSZ-CC-0666-2012,recertification,None +259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None +25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None +25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None +26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,None +26c7b173668167d8,BSI-DSZ-CC-0813-2012,component_used,None +2760aeedce0b79db,ANSSI-CC-2011/64,evaluation_reused,None +2760aeedce0b79db,BSI-DSZ-CC-0555-2009,component_used,None +27ccec0740bb7915,ANSSI-CC-2010/02,component_used,None +27ccec0740bb7915,ANSSI-CC-2010/20,evaluation_reused,None +27ccec0740bb7915,ANSSI-CC-2010/35,evaluation_reused,None +296618f10b019d9c,BSI-DSZ-CC-0417-2008,evaluation_reused,None +29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None +29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None +29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,recertification,None +2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None +2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None +2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,recertification,None +2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None +2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None +2bd9661657e78278,ANSSI-CC-2013/13,component_used,None +2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,recertification,None +2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,recertification,None +2d046178aa118fff,BSI-DSZ-CC-0293-2005,recertification,None +2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None +2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None +2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,recertification,None +30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None +3183669bf78db43b,ANSSI-CC-2015/45,component_used,None +3183669bf78db43b,ANSSI-CC-2016/64,component_used,None +3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,component_used,None +31fd07069aebf013,BSI-DSZ-CC-0466-2008,evaluation_reused,None +328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None +328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None +3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,recertification,None +3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None +337ece90615ed69d,ANSSI-CC-2010/02,component_used,None +3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None +33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None +33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,recertification,None +34506b8fab6dd7c3,ANSSI-CC-2014/22,evaluation_reused,None +34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None +34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None +3519c14e4114d93d,BSI-DSZ-CC-0782-2012,component_used,None +35b9be2203d2b9e6,ANSSI-CC-2012/70,component_used,None +35efe13fa9e93a68,CRP178,component_used,None +35efe13fa9e93a68,CRP182,component_used,None +36641ce7192b5f92,BSI-DSZ-CC-0977-2017,component_used,None +054bd8196e015197,ANSSI-CC-2011/07,None,unclear +122245dd15683e26,BSI-DSZ-CC-0269-2006,None,unclear +25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,component_used,very difficult +25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,component_used,None +29cc495628e76ffa,BSI-DSZ-CC-0886-2013,irrelevant,None +2a07b85f61ac08b0,ANSSI-CC-2019/02,component_used,None +30c6777901ae346c,BSI-DSZ-CC-0438-2007,component_used,None +35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,irrelevant,None +36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,component_used,None +373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,component_used,None +3746512864b941d4,ANSSI-CC-2016/15,evaluation_reused,None +37b94c08f8e25249,BSI-DSZ-CC-0404-2007,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0410-2007,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0555-2009,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0633-2010,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0674-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0675-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0709-2010,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0710-2010,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0730-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0797-2012,irrelevant,None +37b94c08f8e25249,BSI-DSZ-CC-0799-2012,irrelevant,None +37c8d23c44b95833,ANSSI-CC-2017/16,evaluation_reused,None +37d28734245a3bb1,BSI-DSZ-CC-0257-2004,component_used,None +37e0d3cb098458d0,ANSSI-CC-2014/50,None,unclear +37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,component_used,None +37fe6036c2ac932b,ANSSI-CC-2014/06,component_used,None +37fe6036c2ac932b,ANSSI-CC-2014/07,component_shared,None +37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,component_used,None +3817c0aca007989e,ANSSI-CC-2011/10,component_used,None +3817c0aca007989e,ANSSI-CC-2011/12,component_shared,None +3817c0aca007989e,BSI-DSZ-CC-0523-2008,component_used,None +3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None +384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,None,unclear +384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None +3976c9e492193315,BSI-DSZ-CC-1113-2021,recertification,None +39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None +3a3b0a9113835307,BSI-DSZ-CC-0402-2008,recertification,None +3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None +3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None +3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None +3a6a5f536bfbd4d4,ANSSI-CC-2009/59,previous_version,None +3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,component_used,None +3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,evaluation_reused,None +3d149fe7c08bfc58,CRP256,previous_version,None +3d756c83419bba28,ANSSI-CC-2011/07,component_used,None +3d756c83419bba28,ANSSI-CC-2011/77,previous_version,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,component_used,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,component_used,None +3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,component_used,None +3e08a27e9d9c9b1e,NSCIB-CC-16-99111,irrelevant,self reference +3e182053b03f1faf,CCEVS-VR-06-0044,irrelevant,self reference +3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,component_used,None +3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None +3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None +3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None +3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,recertification,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,recertification,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None +3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None +3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,recertification,None +3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None +3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None +4057f5dbe4ffa4cd,ANSSI-CC-2017/57,evaluation_reused,None +40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,recertification,None +40fa0a4cb5193977,ANSSI-CC-2018/27,evaluation_reused,None +40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None +40fc6ad0aed92913,ANSSI-CC-2017/54,evaluation_reused,None +40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,component_used,None +41bd2924e9afced5,ANSSI-CC-2010/49,component_used,None +41bd2924e9afced5,ANSSI-CC-2010/50,evaluation_reused,None +420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None +424f6d496746258b,ANSSI-CC-2019/12,component_used,None +424f6d496746258b,BSI-DSZ-CC-0995-2018,recertification,None +4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None +42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None +42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,None +42f53e51476f1a3c,ANSSI-CC-2011/07,previous_version,None +438b59086f6ebd64,BSI-DSZ-CC-1107-2020,component_used,None +43a4ca62d0c0b0da,ANSSI-CC-2017/63,evaluation_reused,None +43a4ca62d0c0b0da,ANSSI-CC-2017/64,component_used,None +43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,component_used,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,component_used,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,component_used,None +449c74a92ebb61a4,ANSSI-CC-2017/24,component_used,None +44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,component_used,None +45098872448f5816,NSCIB-CC-0441513-CR,component_used,None +46459cfb9c1045d8,ANSSI-CC-2018/27,component_used,None +46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,component_used,None +4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,previous_version,None +47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None +4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None +4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,recertification,None +4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None +4893140ae5daadaf,CRP278,component_used,None +48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,recertification,None +48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None +48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None +491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None +491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,recertification,None +49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None +49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None +49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None +4abc83fee59586b2,ANSSI-CC-2020/07,component_used,None +4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,component_used,None +4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None +4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None +4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None +4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,recertification,None diff --git a/src/sec_certs/data/reference_annotations/adam/train.csv b/src/sec_certs/data/reference_annotations/adam/train.csv new file mode 100644 index 00000000..6da4fc05 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/adam/train.csv @@ -0,0 +1,101 @@ +dgst,canonical_reference_keyword,label,comment +e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified M7892 G12 component +0bf7a19b22163465,ANSSI-CC-2016/44,evaluation_reused,unclear whether component used +ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None +8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None +680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None +3515801dee00995f,BSI-DSZ-CC-0447-2008,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used +d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert +921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned +00efeb17bcaafce6,ANSSI-CC-2009/34,component_used,None +c175537338906951,ANSSI-CC-2016/70,component_shared,None +fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None +ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None +dba4b5a166f9456a,ANSSI-CC-2014/76,evaluation_reused,None +aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None +b42009eb34746731,ANSSI-CC-2021/50,previous_version,maybe previous_version +463ecd64b7506048,ANSSI-CC-2018/52,component_used,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,recertification,None +c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None +0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None +2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None +e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,unclear +ee1c6dd97918d74a,ANSSI-CC-2012/71,evaluation_reused,None +e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,component_used,None +845bb039719ac5d8,ANSSI-CC-2010/40,previous_version,None +af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None +d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None +54273dd266fce692,ANSSI-CC-2015/15,component_used,None +4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,recertification,None +a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None +f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None +8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None +b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,unclear +371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,recertification,None +4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,recertification,None +16bdbde359584f99,ANSSI-CC-2017/61,component_used,None +34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None +a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None +0bf7a19b22163465,ANSSI-CC-2015/80,evaluation_reused,None +1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,component_used,None +13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,component_shared,"BAC, EAC thingy" +f869956d14ea1694,ANSSI-CC-2012/68,component_used,None +037577fc2019fcfa,ANSSI-CC-2021/49,previous_version,None +17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,component_used,None +5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,component_used,None +1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,component_used,None +28e70f46ba4394dd,ANSSI-CC-2012/08,component_used,None +61832cb4291c343f,BSI-DSZ-CC-0955-2016,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0312-2005,component_used,None +627bfd69c2a831cf,ANSSI-CC-2014/37,evaluation_reused,None +aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 +05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None +9529eb793550093c,ANSSI-CC-2018/40,previous_version,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,recertification,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,recertification,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,recertification,None +ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None +30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None +6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None +06e505abb8dad1b8,ANSSI-CC-2017/54,component_used,None +317713948b4473b3,BSI-DSZ-CC-0798-2012,component_shared,None +055985699aee1e09,ANSSI-CC-2010/02,component_used,None +b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None +691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,recertification,None +19c60923e31777e7,ANSSI-CC-2017/49,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None +c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,recertification,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,recertification,None +5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear +4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None +c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None +ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None +30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,component_used,None +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,recertification,None +ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None +bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" +97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None +744a7a202d909323,ANSSI-CC-2017/24,component_used,None +df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None +6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None +1b8705a0486b3be1,NSCIB-CC-0209053-CR,previous_version,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,recertification,None +cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None +27d11629261d8806,ANSSI-CC-2014/59,component_used,None +a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,None +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,recertification,None +c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery" +84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None +36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None +01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,component_used,None +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,component_used,None diff --git a/src/sec_certs/data/reference_annotations/adam/valid.csv b/src/sec_certs/data/reference_annotations/adam/valid.csv new file mode 100644 index 00000000..948845eb --- /dev/null +++ b/src/sec_certs/data/reference_annotations/adam/valid.csv @@ -0,0 +1,108 @@ +dgst,canonical_reference_keyword,label,comment +a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,recertification,None +0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,recertification,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,recertification,None +774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,None +c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,evaluation_reused,None +a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None +4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,component_used,None +cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None +24de96ea505af909,ANSSI-CC-2021/29,component_used,None +2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,recertification,None +d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None +4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,None +22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,recertification,None +6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None +120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None +994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None +16abf8ee0697e64f,ANSSI-CC-2014/61,component_shared,None +f05de4db911ec8af,CRP243,previous_version,None +38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None +fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,recertification,None +c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_used,None +2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None +833ff87d69cff5db,383-4-184,None,unclear +c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,recertification,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,recertification,None +94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None +37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None +5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,recertification,None +d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None +0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,unclear +84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,recertification,None +d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,recertification,None +e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None +db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None +4b3f577896c80f8a,ANSSI-CC-2016/44,previous_version,None +5f3c9c6bf76ba72d,ANSSI-CC-2018/51,component_used,None +22d6578b30f5a227,BSI-DSZ-CC-1107-2020,component_used,None +a0c38b4389ad7cf7,ANSSI-CC-2018/12,component_used,None +19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,component_used,None +f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,unclear +2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,component_used,None +3357c28c91cc2ed1,ANSSI-CC-2016/44,evaluation_reused,None +f6317f134f532696,ANSSI-CC-2017/24,component_shared,None +234da54efbcf734b,BSI-DSZ-CC-0917-2014,component_used,None +1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0633-2010,evaluation_reused,None +b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,recertification,None +edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,component_shared,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. +Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of +technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the +TOE was re-evaluated because of changes on top layers for modified module +packaging requirements. Additional package types (MOB4 and complete +passport inlay) and the relevant production sites were included in the evaluation +process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the +TOE itself. The Security Target was updated. " +3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_shared,None +0fe8df0e85116b61,ANSSI-CC-2010/33,component_shared,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,recertification,None +b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,None +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,recertification,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,recertification,None +63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None +a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None +4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None +f4d510a39278e687,ANSSI-CC-2018/32,component_used,None +c23bb542201aa511,ANSSI-CC-2014/14,component_shared,None +625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,recertification,None +000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None +66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None +a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None +031667f4e242da61,CCEVS-VR-07-0054,component_used,None +459539da9315ccac,ANSSI-CC-2021/30,previous_version,None +69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None +a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,recertification,None +1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None +0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None +8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None +6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None +4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None +e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,recertification,None +720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,recertification,None +d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None +73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None +2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None +f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,component_used,None +f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,component_used,None +2f43fe24bd8e91e1,ANSSI-CC-2020/65,component_used,None +d96438a7165e5d82,BSI-DSZ-CC-0411-2007,component_used,None +7623950728f52a71,ANSSI-CC-2016/06,component_used,None +ba69597434946260,BSI-DSZ-CC-0293-2005,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-2018,component_used,None diff --git a/src/sec_certs/data/reference_annotations/conflicts/test.csv b/src/sec_certs/data/reference_annotations/conflicts/test.csv new file mode 100644 index 00000000..dca3c452 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/conflicts/test.csv @@ -0,0 +1,47 @@ +dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label +173097995b7a7f12,ANSSI-CC-2012/49,component_shared,evaluation_reused,component_shared +173097995b7a7f12,ANSSI-CC-2013/28,evaluation_reused,component_used,component_used +18b7e5bdf459ca13,NSCIB-CC-07-09219,component_used,previous_version,previous_version +195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None,component_used +195366bfba7213a5,BSI-DSZ-CC-0322-2005,component_used,None,previous_version +195366bfba7213a5,BSI-DSZ-CC-0399-2007,recertification,None,recertification +1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None,component_used +1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,evaluation_reused,component_shared,component_shared +22ede463dbf1a105,ANSSI-CC-2015/36,evaluation_reused,component_used,component_used +22ede463dbf1a105,ANSSI-CC-2015/80,evaluation_reused,irrelevant,irrelevant +26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,component_used,component_used +27ccec0740bb7915,ANSSI-CC-2010/35,evaluation_reused,component_shared,component_shared +296618f10b019d9c,BSI-DSZ-CC-0417-2008,evaluation_reused,component_used,component_used +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,recertification,evaluation_reused,recertification +31fd07069aebf013,BSI-DSZ-CC-0466-2008,evaluation_reused,component_used,previous_version +34506b8fab6dd7c3,ANSSI-CC-2014/22,evaluation_reused,previous_version,previous_version +054bd8196e015197,ANSSI-CC-2011/07,None,evaluation_reused,evaluation_reused +25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,component_used,None,component_used +25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,component_used,evaluation_reused,component_used +29cc495628e76ffa,BSI-DSZ-CC-0886-2013,irrelevant,component_shared,component_shared +35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,irrelevant,component_shared,component_shared +3746512864b941d4,ANSSI-CC-2016/15,evaluation_reused,previous_version,evaluation_reused +37b94c08f8e25249,BSI-DSZ-CC-0404-2007,component_used,None,component_used +37b94c08f8e25249,BSI-DSZ-CC-0410-2007,component_used,None,component_used +37b94c08f8e25249,BSI-DSZ-CC-0555-2009,component_used,None,component_used +37b94c08f8e25249,BSI-DSZ-CC-0633-2010,component_used,None,component_used +37b94c08f8e25249,BSI-DSZ-CC-0709-2010,component_used,None,component_used +37b94c08f8e25249,BSI-DSZ-CC-0710-2010,component_used,None,component_used +37b94c08f8e25249,BSI-DSZ-CC-0797-2012,irrelevant,component_shared,component_shared +37b94c08f8e25249,BSI-DSZ-CC-0799-2012,irrelevant,None,component_shared +37c8d23c44b95833,ANSSI-CC-2017/16,evaluation_reused,previous_version,previous_version +37d28734245a3bb1,BSI-DSZ-CC-0257-2004,component_used,previous_version,component_used +37e0d3cb098458d0,ANSSI-CC-2014/50,None,component_used,component_used +37fe6036c2ac932b,ANSSI-CC-2014/07,component_shared,previous_version,previous_version +3817c0aca007989e,ANSSI-CC-2011/12,component_shared,previous_version,component_shared +384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,None,component_used,component_used +3e08a27e9d9c9b1e,NSCIB-CC-16-99111,irrelevant,previous_version,irrelevant +4057f5dbe4ffa4cd,ANSSI-CC-2017/57,evaluation_reused,previous_version,previous_version +40fa0a4cb5193977,ANSSI-CC-2018/27,evaluation_reused,previous_version,component_shared +40fc6ad0aed92913,ANSSI-CC-2017/54,evaluation_reused,component_used,previous_version +41bd2924e9afced5,ANSSI-CC-2010/49,component_used,previous_version,previous_version +41bd2924e9afced5,ANSSI-CC-2010/50,evaluation_reused,previous_version,previous_version +42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,previous_version,evaluation_reused +43a4ca62d0c0b0da,ANSSI-CC-2017/64,component_used,irrelevant,component_used +4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,previous_version,irrelevant,irrelevant +4893140ae5daadaf,CRP278,component_used,component_shared,component_used diff --git a/src/sec_certs/data/reference_annotations/conflicts/train.csv b/src/sec_certs/data/reference_annotations/conflicts/train.csv new file mode 100644 index 00000000..8ac397ce --- /dev/null +++ b/src/sec_certs/data/reference_annotations/conflicts/train.csv @@ -0,0 +1,10 @@ +dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label +037577fc2019fcfa,ANSSI-CC-2021/49,previous_version,evaluation_reused,previous_version +1b8705a0486b3be1,NSCIB-CC-0209053-CR,previous_version,evaluation_reused,previous_version +9529eb793550093c,ANSSI-CC-2018/40,previous_version,evaluation_reused,previous_version +a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,evaluation_reused,component_used +b42009eb34746731,ANSSI-CC-2021/50,previous_version,evaluation_reused,previous_version +bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,None,evaluation_reused +c175537338906951,ANSSI-CC-2016/70,component_shared,component_used,component_shared +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,recertification,evaluation_reused,evaluation_reused +dba4b5a166f9456a,ANSSI-CC-2014/76,evaluation_reused,previous_version,recertification diff --git a/src/sec_certs/data/reference_annotations/conflicts/valid.csv b/src/sec_certs/data/reference_annotations/conflicts/valid.csv new file mode 100644 index 00000000..b18f88ee --- /dev/null +++ b/src/sec_certs/data/reference_annotations/conflicts/valid.csv @@ -0,0 +1,17 @@ +dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label +0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,None,component_used +0fe8df0e85116b61,ANSSI-CC-2010/33,component_shared,evaluation_reused,previous_version +16abf8ee0697e64f,ANSSI-CC-2014/61,component_shared,evaluation_reused,component_shared +2532fe8c7214c485,BSI-DSZ-CC-0633-2010,evaluation_reused,component_used,component_used +3357c28c91cc2ed1,ANSSI-CC-2016/44,evaluation_reused,component_used,evaluation_reused +3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_shared,component_used,component_used +4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,component_used,evaluation_reused +4b3f577896c80f8a,ANSSI-CC-2016/44,previous_version,component_used,evaluation_reused +774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,component_used,evaluation_reused +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,recertification,previous_version,recertification +b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,evaluation_reused,previous_version +c23bb542201aa511,ANSSI-CC-2014/14,component_shared,component_used,previous_version +c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,evaluation_reused,component_shared,None +edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,component_shared,evaluation_reused,evaluation_reused +f05de4db911ec8af,CRP243,previous_version,component_used,previous_version +f6317f134f532696,ANSSI-CC-2017/24,component_shared,evaluation_reused,previous_version diff --git a/src/sec_certs/data/reference_annotations/final/test.csv b/src/sec_certs/data/reference_annotations/final/test.csv new file mode 100644 index 00000000..fbda9fd8 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/final/test.csv @@ -0,0 +1,201 @@ +dgst,canonical_reference_keyword,label,comment +15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,recertification,None +15b121492722bffb,BSI-DSZ-CC-0754-2012,recertification,None +16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,recertification,None +16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None +16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None +173097995b7a7f12,ANSSI-CC-2012/49,component_shared,None +173097995b7a7f12,ANSSI-CC-2013/28,component_used,None +173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None +173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None +177dee4c1051e612,BSI-DSZ-CC-0854-2013,recertification,None +17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None +183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None +183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None +18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,component_used,None +18b7e5bdf459ca13,NSCIB-CC-07-09219,previous_version,None +195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None +195366bfba7213a5,BSI-DSZ-CC-0322-2005,previous_version,None +195366bfba7213a5,BSI-DSZ-CC-0399-2007,recertification,None +1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None +19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None +1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,recertification,None +1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None +1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None +1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None +1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None +1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,recertification,None +1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,None +1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,component_shared,None +1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,recertification,None +1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None +1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,recertification,None +2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,recertification,None +21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None +21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None +22ede463dbf1a105,ANSSI-CC-2015/36,component_used,None +22ede463dbf1a105,ANSSI-CC-2015/80,irrelevant,None +22ede463dbf1a105,ANSSI-CC-2016/43,evaluation_reused,None +22ede463dbf1a105,ANSSI-CC-2016/44,evaluation_reused,None +23a85a7f9c07412f,ANSSI-CC-2014/46,evaluation_reused,None +23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,None +24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None +24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None +259df98e7476a843,BSI-DSZ-CC-0666-2012,recertification,None +259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None +25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None +25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None +26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,component_used,None +26c7b173668167d8,BSI-DSZ-CC-0813-2012,component_used,None +2760aeedce0b79db,ANSSI-CC-2011/64,evaluation_reused,None +2760aeedce0b79db,BSI-DSZ-CC-0555-2009,component_used,None +27ccec0740bb7915,ANSSI-CC-2010/02,component_used,None +27ccec0740bb7915,ANSSI-CC-2010/20,evaluation_reused,None +27ccec0740bb7915,ANSSI-CC-2010/35,component_shared,None +296618f10b019d9c,BSI-DSZ-CC-0417-2008,component_used,None +29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None +29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None +29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,recertification,None +2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None +2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None +2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,recertification,None +2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None +2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None +2bd9661657e78278,ANSSI-CC-2013/13,component_used,None +2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,recertification,None +2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,recertification,None +2d046178aa118fff,BSI-DSZ-CC-0293-2005,recertification,None +2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None +2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None +2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,recertification,None +30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None +3183669bf78db43b,ANSSI-CC-2015/45,component_used,None +3183669bf78db43b,ANSSI-CC-2016/64,component_used,None +3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,component_used,None +31fd07069aebf013,BSI-DSZ-CC-0466-2008,previous_version,None +328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None +328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None +3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,recertification,None +3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None +337ece90615ed69d,ANSSI-CC-2010/02,component_used,None +3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None +33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None +33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,recertification,None +34506b8fab6dd7c3,ANSSI-CC-2014/22,previous_version,None +34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None +34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None +3519c14e4114d93d,BSI-DSZ-CC-0782-2012,component_used,None +35b9be2203d2b9e6,ANSSI-CC-2012/70,component_used,None +35efe13fa9e93a68,CRP178,component_used,None +35efe13fa9e93a68,CRP182,component_used,None +36641ce7192b5f92,BSI-DSZ-CC-0977-2017,component_used,None +054bd8196e015197,ANSSI-CC-2011/07,evaluation_reused,unclear +122245dd15683e26,BSI-DSZ-CC-0269-2006,None,unclear +25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,component_used,very difficult +25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,component_used,None +29cc495628e76ffa,BSI-DSZ-CC-0886-2013,component_shared,None +2a07b85f61ac08b0,ANSSI-CC-2019/02,component_used,None +30c6777901ae346c,BSI-DSZ-CC-0438-2007,component_used,None +35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,component_shared,None +36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,component_used,None +373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,component_used,None +3746512864b941d4,ANSSI-CC-2016/15,evaluation_reused,None +37b94c08f8e25249,BSI-DSZ-CC-0404-2007,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0410-2007,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0555-2009,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0633-2010,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0674-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0675-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0709-2010,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0710-2010,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0730-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0797-2012,component_shared,None +37b94c08f8e25249,BSI-DSZ-CC-0799-2012,component_shared,None +37c8d23c44b95833,ANSSI-CC-2017/16,previous_version,None +37d28734245a3bb1,BSI-DSZ-CC-0257-2004,component_used,None +37e0d3cb098458d0,ANSSI-CC-2014/50,component_used,unclear +37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,component_used,None +37fe6036c2ac932b,ANSSI-CC-2014/06,component_used,None +37fe6036c2ac932b,ANSSI-CC-2014/07,previous_version,None +37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,component_used,None +3817c0aca007989e,ANSSI-CC-2011/10,component_used,None +3817c0aca007989e,ANSSI-CC-2011/12,component_shared,None +3817c0aca007989e,BSI-DSZ-CC-0523-2008,component_used,None +3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None +384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,component_used,unclear +384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None +3976c9e492193315,BSI-DSZ-CC-1113-2021,recertification,None +39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None +3a3b0a9113835307,BSI-DSZ-CC-0402-2008,recertification,None +3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None +3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None +3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None +3a6a5f536bfbd4d4,ANSSI-CC-2009/59,previous_version,None +3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,component_used,None +3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,evaluation_reused,None +3d149fe7c08bfc58,CRP256,previous_version,None +3d756c83419bba28,ANSSI-CC-2011/07,component_used,None +3d756c83419bba28,ANSSI-CC-2011/77,previous_version,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,component_used,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,component_used,None +3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,component_used,None +3e08a27e9d9c9b1e,NSCIB-CC-16-99111,irrelevant,self reference +3e182053b03f1faf,CCEVS-VR-06-0044,irrelevant,self reference +3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,component_used,None +3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None +3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None +3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None +3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,recertification,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,recertification,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None +3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None +3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,recertification,None +3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None +3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None +4057f5dbe4ffa4cd,ANSSI-CC-2017/57,previous_version,None +40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,recertification,None +40fa0a4cb5193977,ANSSI-CC-2018/27,component_shared,None +40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None +40fc6ad0aed92913,ANSSI-CC-2017/54,previous_version,None +40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,component_used,None +41bd2924e9afced5,ANSSI-CC-2010/49,previous_version,None +41bd2924e9afced5,ANSSI-CC-2010/50,previous_version,None +420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None +424f6d496746258b,ANSSI-CC-2019/12,component_used,None +424f6d496746258b,BSI-DSZ-CC-0995-2018,recertification,None +4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None +42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None +42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,None +42f53e51476f1a3c,ANSSI-CC-2011/07,previous_version,None +438b59086f6ebd64,BSI-DSZ-CC-1107-2020,component_used,None +43a4ca62d0c0b0da,ANSSI-CC-2017/63,evaluation_reused,None +43a4ca62d0c0b0da,ANSSI-CC-2017/64,component_used,None +43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,component_used,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,component_used,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,component_used,None +449c74a92ebb61a4,ANSSI-CC-2017/24,component_used,None +44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,component_used,None +45098872448f5816,NSCIB-CC-0441513-CR,component_used,None +46459cfb9c1045d8,ANSSI-CC-2018/27,component_used,None +46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,component_used,None +4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,irrelevant,None +47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None +4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None +4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,recertification,None +4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None +4893140ae5daadaf,CRP278,component_used,None +48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,recertification,None +48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None +48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None +491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None +491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,recertification,None +49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None +49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None +49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None +4abc83fee59586b2,ANSSI-CC-2020/07,component_used,None +4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,component_used,None +4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None +4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None +4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None +4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,recertification,None diff --git a/src/sec_certs/data/reference_annotations/final/train.csv b/src/sec_certs/data/reference_annotations/final/train.csv new file mode 100644 index 00000000..b6b81d30 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/final/train.csv @@ -0,0 +1,101 @@ +dgst,canonical_reference_keyword,label,comment +00efeb17bcaafce6,ANSSI-CC-2009/34,component_used,None +01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,component_used,None +037577fc2019fcfa,ANSSI-CC-2021/49,previous_version,None +055985699aee1e09,ANSSI-CC-2010/02,component_used,None +05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None +06e505abb8dad1b8,ANSSI-CC-2017/54,component_used,None +0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None +0bf7a19b22163465,ANSSI-CC-2015/80,evaluation_reused,None +0bf7a19b22163465,ANSSI-CC-2016/44,evaluation_reused,unclear whether component used +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,component_used,None +12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used +13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,component_shared,"BAC, EAC thingy" +16bdbde359584f99,ANSSI-CC-2017/61,component_used,None +17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,component_used,None +19c60923e31777e7,ANSSI-CC-2017/49,component_used,None +1b8705a0486b3be1,NSCIB-CC-0209053-CR,previous_version,None +1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,component_used,None +1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,component_used,None +27d11629261d8806,ANSSI-CC-2014/59,component_used,None +28e70f46ba4394dd,ANSSI-CC-2012/08,component_used,None +2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None +30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,component_used,None +30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None +317713948b4473b3,BSI-DSZ-CC-0798-2012,component_shared,None +34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None +3515801dee00995f,BSI-DSZ-CC-0447-2008,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 +36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None +371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,recertification,None +42799014c183e1b3,BSI-DSZ-CC-0312-2005,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,recertification,None +463ecd64b7506048,ANSSI-CC-2018/52,component_used,None +4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None +4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None +4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None +54273dd266fce692,ANSSI-CC-2015/15,component_used,None +5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,component_used,None +5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear +61832cb4291c343f,BSI-DSZ-CC-0955-2016,component_used,None +627bfd69c2a831cf,ANSSI-CC-2014/37,evaluation_reused,None +680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None +691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None +6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None +744a7a202d909323,ANSSI-CC-2017/24,component_used,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,recertification,None +845bb039719ac5d8,ANSSI-CC-2010/40,previous_version,None +84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None +8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None +8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None +921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned +9529eb793550093c,ANSSI-CC-2018/40,previous_version,None +97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,recertification,None +a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None +a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None +a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,recertification,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,recertification,None +aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None +ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None +ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None +aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 +af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None +b42009eb34746731,ANSSI-CC-2021/50,previous_version,maybe previous_version +b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None +bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" +c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None +c175537338906951,ANSSI-CC-2016/70,component_shared,None +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,recertification,None +c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None +c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None +cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None +cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None +d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,recertification,None +d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,recertification,None +dba4b5a166f9456a,ANSSI-CC-2014/76,recertification,None +df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,recertification,None +e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified M7892 G12 component +e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,component_used,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,recertification,None +ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None +ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None +ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,recertification,None +ee1c6dd97918d74a,ANSSI-CC-2012/71,evaluation_reused,None +f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 +f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +f869956d14ea1694,ANSSI-CC-2012/68,component_used,None +fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None +b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,irrelevant,None +c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery" +e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,irrelevant,unclear diff --git a/src/sec_certs/data/reference_annotations/final/valid.csv b/src/sec_certs/data/reference_annotations/final/valid.csv new file mode 100644 index 00000000..a74d7a45 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/final/valid.csv @@ -0,0 +1,108 @@ +dgst,canonical_reference_keyword,label,comment +a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,recertification,None +0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,recertification,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,recertification,None +774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,None +c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,None,None +a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None +4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,component_used,None +cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None +24de96ea505af909,ANSSI-CC-2021/29,component_used,None +2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,recertification,None +d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None +4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,None +22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,recertification,None +6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None +120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None +994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None +16abf8ee0697e64f,ANSSI-CC-2014/61,component_shared,None +f05de4db911ec8af,CRP243,previous_version,None +38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None +fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,recertification,None +c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_shared,None +2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None +833ff87d69cff5db,383-4-184,None,unclear +c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,recertification,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,recertification,None +94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None +37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None +5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,recertification,None +d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None +0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,unclear +84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,recertification,None +d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,recertification,None +e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None +db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None +4b3f577896c80f8a,ANSSI-CC-2016/44,evaluation_reused,None +5f3c9c6bf76ba72d,ANSSI-CC-2018/51,component_used,None +22d6578b30f5a227,BSI-DSZ-CC-1107-2020,component_used,None +a0c38b4389ad7cf7,ANSSI-CC-2018/12,component_used,None +19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,component_used,None +f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,unclear +2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,component_used,None +3357c28c91cc2ed1,ANSSI-CC-2016/44,evaluation_reused,None +f6317f134f532696,ANSSI-CC-2017/24,previous_version,None +234da54efbcf734b,BSI-DSZ-CC-0917-2014,component_used,None +1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0633-2010,component_used,None +b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,recertification,None +edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. +Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of +technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the +TOE was re-evaluated because of changes on top layers for modified module +packaging requirements. Additional package types (MOB4 and complete +passport inlay) and the relevant production sites were included in the evaluation +process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the +TOE itself. The Security Target was updated. " +3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_used,None +0fe8df0e85116b61,ANSSI-CC-2010/33,previous_version,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,recertification,None +b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,None +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,recertification,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,recertification,None +63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None +a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None +4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None +f4d510a39278e687,ANSSI-CC-2018/32,component_used,None +c23bb542201aa511,ANSSI-CC-2014/14,previous_version,None +625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,recertification,None +000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None +66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None +a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None +031667f4e242da61,CCEVS-VR-07-0054,component_used,None +459539da9315ccac,ANSSI-CC-2021/30,previous_version,None +69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None +a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,recertification,None +1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None +0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None +8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None +6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None +4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None +e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,recertification,None +720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,recertification,None +d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None +73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None +2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None +f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,component_used,None +f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,component_used,None +2f43fe24bd8e91e1,ANSSI-CC-2020/65,component_used,None +d96438a7165e5d82,BSI-DSZ-CC-0411-2007,component_used,None +7623950728f52a71,ANSSI-CC-2016/06,component_used,None +ba69597434946260,BSI-DSZ-CC-0293-2005,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-2018,component_used,None diff --git a/src/sec_certs/data/reference_annotations/jano/test.csv b/src/sec_certs/data/reference_annotations/jano/test.csv new file mode 100644 index 00000000..b241171b --- /dev/null +++ b/src/sec_certs/data/reference_annotations/jano/test.csv @@ -0,0 +1,201 @@ +dgst,canonical_reference_keyword,label,comment +15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,recertification,None +15b121492722bffb,BSI-DSZ-CC-0754-2012,recertification,None +16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,recertification,evaluation_reused more like +16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None +16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None +173097995b7a7f12,ANSSI-CC-2012/49,evaluation_reused,None +173097995b7a7f12,ANSSI-CC-2013/28,component_used,None +173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None +173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None +177dee4c1051e612,BSI-DSZ-CC-0854-2013,recertification,None +17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None +183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None +183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None +18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,component_used,None +18b7e5bdf459ca13,NSCIB-CC-07-09219,previous_version,also evaluation_reused +195366bfba7213a5,BSI-DSZ-CC-0266-2005,None,None +195366bfba7213a5,BSI-DSZ-CC-0322-2005,None,None +195366bfba7213a5,BSI-DSZ-CC-0399-2007,None,None +1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,None,None +19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None +1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,recertification,None +1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None +1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None +1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None +1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None +1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,recertification,None +1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,also previous version to a degree +1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,component_shared,None +1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,recertification,None +1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None +1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,recertification,None +2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,recertification,None +21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None +21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None +22ede463dbf1a105,ANSSI-CC-2015/36,component_used,None +22ede463dbf1a105,ANSSI-CC-2015/80,irrelevant,mistake in report +22ede463dbf1a105,ANSSI-CC-2016/43,evaluation_reused,None +22ede463dbf1a105,ANSSI-CC-2016/44,evaluation_reused,None +23a85a7f9c07412f,ANSSI-CC-2014/46,evaluation_reused,component_used probably +23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,component_used probably +24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None +24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None +259df98e7476a843,BSI-DSZ-CC-0666-2012,recertification,None +259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None +25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None +25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None +26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,component_used,None +26c7b173668167d8,BSI-DSZ-CC-0813-2012,component_used,None +2760aeedce0b79db,ANSSI-CC-2011/64,evaluation_reused,None +2760aeedce0b79db,BSI-DSZ-CC-0555-2009,component_used,None +27ccec0740bb7915,ANSSI-CC-2010/02,component_used,None +27ccec0740bb7915,ANSSI-CC-2010/20,evaluation_reused,None +27ccec0740bb7915,ANSSI-CC-2010/35,component_shared,None +296618f10b019d9c,BSI-DSZ-CC-0417-2008,component_used,None +29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None +29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None +29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,recertification,None +2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None +2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None +2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,recertification,None +2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None +2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None +2bd9661657e78278,ANSSI-CC-2013/13,component_used,None +2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,recertification,None +2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,recertification,None +2d046178aa118fff,BSI-DSZ-CC-0293-2005,recertification,None +2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None +2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None +2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,evaluation_reused,maybe recertification? +30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None +3183669bf78db43b,ANSSI-CC-2015/45,component_used,None +3183669bf78db43b,ANSSI-CC-2016/64,component_used,None +3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,component_used,None +31fd07069aebf013,BSI-DSZ-CC-0466-2008,component_used,None +328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None +328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None +3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,recertification,None +3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None +337ece90615ed69d,ANSSI-CC-2010/02,component_used,None +3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None +33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None +33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,recertification,None +34506b8fab6dd7c3,ANSSI-CC-2014/22,previous_version,None +34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None +34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None +3519c14e4114d93d,BSI-DSZ-CC-0782-2012,component_used,None +35b9be2203d2b9e6,ANSSI-CC-2012/70,component_used,None +35efe13fa9e93a68,CRP178,component_used,None +35efe13fa9e93a68,CRP182,component_used,None +36641ce7192b5f92,BSI-DSZ-CC-0977-2017,component_used,None +054bd8196e015197,ANSSI-CC-2011/07,evaluation_reused,None +122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None +25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,None,None +25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,evaluation_reused,None +29cc495628e76ffa,BSI-DSZ-CC-0886-2013,component_shared,None +2a07b85f61ac08b0,ANSSI-CC-2019/02,component_used,None +30c6777901ae346c,BSI-DSZ-CC-0438-2007,component_used,None +35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,component_shared,None +36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,component_used,None +373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,component_used,None +3746512864b941d4,ANSSI-CC-2016/15,previous_version,None +37b94c08f8e25249,BSI-DSZ-CC-0404-2007,None,None +37b94c08f8e25249,BSI-DSZ-CC-0410-2007,None,None +37b94c08f8e25249,BSI-DSZ-CC-0555-2009,None,None +37b94c08f8e25249,BSI-DSZ-CC-0633-2010,None,None +37b94c08f8e25249,BSI-DSZ-CC-0674-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0675-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0709-2010,None,None +37b94c08f8e25249,BSI-DSZ-CC-0710-2010,None,None +37b94c08f8e25249,BSI-DSZ-CC-0730-2011,component_used,None +37b94c08f8e25249,BSI-DSZ-CC-0797-2012,component_shared,None +37b94c08f8e25249,BSI-DSZ-CC-0799-2012,None,missing BSI-DSZ-CC-0804-2012 +37c8d23c44b95833,ANSSI-CC-2017/16,previous_version,None +37d28734245a3bb1,BSI-DSZ-CC-0257-2004,previous_version,missing some CRP ids +37e0d3cb098458d0,ANSSI-CC-2014/50,component_used,None +37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,component_used,None +37fe6036c2ac932b,ANSSI-CC-2014/06,component_used,None +37fe6036c2ac932b,ANSSI-CC-2014/07,previous_version,None +37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,component_used,None +3817c0aca007989e,ANSSI-CC-2011/10,component_used,None +3817c0aca007989e,ANSSI-CC-2011/12,previous_version,None +3817c0aca007989e,BSI-DSZ-CC-0523-2008,component_used,None +3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None +384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,component_used,None +384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None +3976c9e492193315,BSI-DSZ-CC-1113-2021,recertification,None +39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None +3a3b0a9113835307,BSI-DSZ-CC-0402-2008,recertification,None +3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None +3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None +3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None +3a6a5f536bfbd4d4,ANSSI-CC-2009/59,previous_version,None +3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,component_used,None +3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,evaluation_reused,None +3d149fe7c08bfc58,CRP256,previous_version,None +3d756c83419bba28,ANSSI-CC-2011/07,component_used,None +3d756c83419bba28,ANSSI-CC-2011/77,previous_version,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,component_used,None +3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,component_used,None +3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,component_used,None +3e08a27e9d9c9b1e,NSCIB-CC-16-99111,previous_version,"but also an issue of cert ID assignment, we need to better canonicalize these “V2” and -CR things" +3e182053b03f1faf,CCEVS-VR-06-0044,irrelevant,None +3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,component_used,None +3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None +3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None +3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None +3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,recertification,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,recertification,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None +3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None +3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,recertification,None +3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None +3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None +4057f5dbe4ffa4cd,ANSSI-CC-2017/57,previous_version,None +40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,recertification,None +40fa0a4cb5193977,ANSSI-CC-2018/27,previous_version,None +40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None +40fc6ad0aed92913,ANSSI-CC-2017/54,component_used,None +40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,component_used,None +41bd2924e9afced5,ANSSI-CC-2010/49,previous_version,None +41bd2924e9afced5,ANSSI-CC-2010/50,previous_version,None +420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None +424f6d496746258b,ANSSI-CC-2019/12,component_used,None +424f6d496746258b,BSI-DSZ-CC-0995-2018,recertification,None +4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None +42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None +42d2e68c29d9eef7,ANSSI-CC-2018/30,previous_version,None +42f53e51476f1a3c,ANSSI-CC-2011/07,previous_version,None +438b59086f6ebd64,BSI-DSZ-CC-1107-2020,component_used,None +43a4ca62d0c0b0da,ANSSI-CC-2017/63,evaluation_reused,None +43a4ca62d0c0b0da,ANSSI-CC-2017/64,irrelevant,"/63 or /64, unsure" +43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,component_used,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,component_used,None +43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,component_used,None +449c74a92ebb61a4,ANSSI-CC-2017/24,component_used,None +44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,component_used,None +45098872448f5816,NSCIB-CC-0441513-CR,component_used,None +46459cfb9c1045d8,ANSSI-CC-2018/27,component_used,None +46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,component_used,None +4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,irrelevant,BSI-DSZ-CC-0891-V4-2018 mentioned in report +47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None +4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None +4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,recertification,None +4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None +4893140ae5daadaf,CRP278,component_shared,None +48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,recertification,None +48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None +48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None +491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None +491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,recertification,None +49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None +49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None +49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None +4abc83fee59586b2,ANSSI-CC-2020/07,component_used,None +4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,component_used,None +4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None +4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None +4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None +4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,recertification,None diff --git a/src/sec_certs/data/reference_annotations/jano/train.csv b/src/sec_certs/data/reference_annotations/jano/train.csv new file mode 100644 index 00000000..18dce096 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/jano/train.csv @@ -0,0 +1,101 @@ +dgst,canonical_reference_keyword,label,comment +e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified M7892 G12 component +0bf7a19b22163465,ANSSI-CC-2016/44,evaluation_reused,unclear whether component used +ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None +8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None +680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None +3515801dee00995f,BSI-DSZ-CC-0447-2008,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used +d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert +921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned +00efeb17bcaafce6,ANSSI-CC-2009/34,component_used,None +c175537338906951,ANSSI-CC-2016/70,component_used,None +fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None +ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None +dba4b5a166f9456a,ANSSI-CC-2014/76,previous_version,None +aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None +b42009eb34746731,ANSSI-CC-2021/50,evaluation_reused,maybe previous_version +463ecd64b7506048,ANSSI-CC-2018/52,component_used,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,recertification,None +c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None +0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None +2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None +e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,unclear +ee1c6dd97918d74a,ANSSI-CC-2012/71,evaluation_reused,None +e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,component_used,None +845bb039719ac5d8,ANSSI-CC-2010/40,previous_version,None +af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None +d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None +54273dd266fce692,ANSSI-CC-2015/15,component_used,None +4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,recertification,None +a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None +f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None +8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None +b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,unclear +371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,recertification,None +4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,recertification,None +16bdbde359584f99,ANSSI-CC-2017/61,component_used,None +34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None +a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None +0bf7a19b22163465,ANSSI-CC-2015/80,evaluation_reused,None +1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,component_used,None +13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,component_shared,"BAC, EAC thingy" +f869956d14ea1694,ANSSI-CC-2012/68,component_used,None +037577fc2019fcfa,ANSSI-CC-2021/49,evaluation_reused,None +17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,component_used,None +5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,component_used,None +1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,component_used,None +28e70f46ba4394dd,ANSSI-CC-2012/08,component_used,None +61832cb4291c343f,BSI-DSZ-CC-0955-2016,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0312-2005,component_used,None +627bfd69c2a831cf,ANSSI-CC-2014/37,evaluation_reused,None +aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 +05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None +9529eb793550093c,ANSSI-CC-2018/40,evaluation_reused,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,recertification,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,recertification,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,recertification,None +ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None +30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None +6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None +06e505abb8dad1b8,ANSSI-CC-2017/54,component_used,None +317713948b4473b3,BSI-DSZ-CC-0798-2012,component_shared,None +055985699aee1e09,ANSSI-CC-2010/02,component_used,None +b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None +691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,recertification,None +19c60923e31777e7,ANSSI-CC-2017/49,component_used,None +f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None +c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,recertification,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,recertification,None +5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear +4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None +c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None +ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None +30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,component_used,None +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,None +ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None +bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,None,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" +97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None +744a7a202d909323,ANSSI-CC-2017/24,component_used,None +df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None +6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None +1b8705a0486b3be1,NSCIB-CC-0209053-CR,evaluation_reused,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,recertification,None +cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None +27d11629261d8806,ANSSI-CC-2014/59,component_used,None +a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,evaluation_reused,None +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,recertification,None +c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery" +84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None +36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None +01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,component_used,None +0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,component_used,None diff --git a/src/sec_certs/data/reference_annotations/jano/valid.csv b/src/sec_certs/data/reference_annotations/jano/valid.csv new file mode 100644 index 00000000..a55055f6 --- /dev/null +++ b/src/sec_certs/data/reference_annotations/jano/valid.csv @@ -0,0 +1,108 @@ +dgst,canonical_reference_keyword,label,comment +a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,recertification,None +0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,recertification,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,recertification,None +774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,component_used,None +c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,component_shared,None +a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None +4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,component_used,None +cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None +24de96ea505af909,ANSSI-CC-2021/29,component_used,None +2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,recertification,None +d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None +4b3f577896c80f8a,ANSSI-CC-2015/80,component_used,None +22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,recertification,None +6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None +120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None +994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None +16abf8ee0697e64f,ANSSI-CC-2014/61,evaluation_reused,None +f05de4db911ec8af,CRP243,component_used,None +38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None +fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,recertification,None +c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_used,None +2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None +833ff87d69cff5db,383-4-184,None,unclear +c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,recertification,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,recertification,None +94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None +37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None +5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,recertification,None +d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None +0f53257eb74bee54,BSI-DSZ-CC-0417-2008,None,unclear +84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,recertification,None +d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,recertification,None +e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None +db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None +4b3f577896c80f8a,ANSSI-CC-2016/44,component_used,None +5f3c9c6bf76ba72d,ANSSI-CC-2018/51,component_used,None +22d6578b30f5a227,BSI-DSZ-CC-1107-2020,component_used,None +a0c38b4389ad7cf7,ANSSI-CC-2018/12,component_used,None +19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,component_used,None +f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,unclear +2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,component_used,None +3357c28c91cc2ed1,ANSSI-CC-2016/44,component_used,None +f6317f134f532696,ANSSI-CC-2017/24,evaluation_reused,None +234da54efbcf734b,BSI-DSZ-CC-0917-2014,component_used,None +1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None +2532fe8c7214c485,BSI-DSZ-CC-0633-2010,component_used,None +b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,recertification,None +edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. +Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of +technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the +TOE was re-evaluated because of changes on top layers for modified module +packaging requirements. Additional package types (MOB4 and complete +passport inlay) and the relevant production sites were included in the evaluation +process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the +TOE itself. The Security Target was updated. " +3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_used,None +0fe8df0e85116b61,ANSSI-CC-2010/33,evaluation_reused,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,recertification,None +b00bb452e7a8176c,ANSSI-CC-2016/57,evaluation_reused,None +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,previous_version,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,recertification,None +63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None +a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None +4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None +f4d510a39278e687,ANSSI-CC-2018/32,component_used,None +c23bb542201aa511,ANSSI-CC-2014/14,component_used,None +625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,recertification,None +000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None +66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None +a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None +031667f4e242da61,CCEVS-VR-07-0054,component_used,None +459539da9315ccac,ANSSI-CC-2021/30,previous_version,None +69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None +a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,recertification,None +1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None +0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None +8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None +6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None +4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None +e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,recertification,None +720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,recertification,None +d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None +73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None +2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None +f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,component_used,None +f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,component_used,None +2f43fe24bd8e91e1,ANSSI-CC-2020/65,component_used,None +d96438a7165e5d82,BSI-DSZ-CC-0411-2007,component_used,None +7623950728f52a71,ANSSI-CC-2016/06,component_used,None +ba69597434946260,BSI-DSZ-CC-0293-2005,component_used,None +50aef770cb4f028f,BSI-DSZ-CC-1059-2018,component_used,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/test.csv b/src/sec_certs/data/reference_annotations/manual_annotations/test.csv deleted file mode 100644 index ad952755..00000000 --- a/src/sec_certs/data/reference_annotations/manual_annotations/test.csv +++ /dev/null @@ -1,1171 +0,0 @@ -dgst,canonical_reference_keyword,source,label,comment -00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None -00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None -00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None -00b539d629e60965,CCEVS-VR-02-0025,None,None,None -00eb3e5c7a4537b1,BSI-DSZ-CC-0266-2005,None,None,None -00eb3e5c7a4537b1,BSI-DSZ-CC-0322-2005,None,None,None -00eb3e5c7a4537b1,BSI-DSZ-CC-0399-2007,None,None,None -017f3cea8275a3e1,ANSSI-CC-2015/59,None,None,None -017f3cea8275a3e1,ANSSI-CC-2017/23,None,None,None -01ce6f422bec823d,BSI-DSZ-CC-1040-2019,None,None,None -01dc77d7f8f94d97,ANSSI-CC-2016/78,None,None,None -01dc77d7f8f94d97,ANSSI-CC-2017/39,None,None,None -01e5c6d31f763f56,BSI-DSZ-CC-0917-2014,None,None,None -0208e6b60a0b1a32,ANSSI-CC-2016/01,None,None,None -02c6fbcd90dcf05a,BSI-DSZ-CC-1107-V3-2022,None,None,None -02cb3c385865ad2c,BSI-DSZ-CC-1184-2022,None,None,None -0319b03323f257d1,BSI-DSZ-CC-1118-2020,None,None,None -0319b03323f257d1,BSI-DSZ-CC-1121-2019,None,None,None -032faef0207b715f,BSI-DSZ-CC-0837-V2-2014,None,None,None -0349c893b81f7c1a,BSI-DSZ-CC-1079-V2-2020,None,None,None -04c5a5a66fb7f89a,BSI-DSZ-CC-0827-V2-2014,None,None,None -04cb360502da6492,BSI-DSZ-CC-0344-2005,None,None,None -054bd8196e015197,ANSSI-CC-2011/07,None,None,None -05d96aada7bb839c,BSI-DSZ-CC-0223-2003,None,None,None -06b24bc51eb69188,BSI-DSZ-CC-0813-2012,None,None,None -06e70fa70393d60f,ANSSI-CC-2017/07,None,None,None -06e70fa70393d60f,BSI-DSZ-CC-0891-V2-2016,None,None,None -075ddd00e07fa7ed,ANSSI-CC-2020/76,None,None,None -075ddd00e07fa7ed,BSI-DSZ-CC-1136-2021,None,None,None -07ce7e4844f62c5c,ANSSI-CC-2017/73,None,None,None -07ce7e4844f62c5c,BSI-DSZ-CC-1070-2018,None,None,None -080aa6966d757f05,ANSSI-CC-2010/02,None,None,None -082380caaa153ee6,BSI-DSZ-CC-0891-V2-2016,None,None,None -08408cbc394e7421,ANSSI-CC-2017/59,None,None,None -08408cbc394e7421,BSI-DSZ-CC-0782-V4-2018,None,None,None -09220515a7edf3f9,BSI-DSZ-CC-1110-V2-2019,None,None,None -096b05114ee5e09f,BSI-DSZ-CC-0410-2007,None,None,None -096b05114ee5e09f,BSI-DSZ-CC-0411-2007,None,None,None -096ccb6bcb3d2731,BSI-DSZ-CC-0564-2009,None,None,None -09b07fbbb24e1b04,BSI-DSZ-CC-0964-2015,None,None,None -09f539aedefa9756,ANSSI-CC-2009/41,None,None,None -0a6e50445a906052,BSI-DSZ-CC-0891-V2-2016,None,None,None -0a6e50445a906052,BSI-DSZ-CC-0945-2017,None,None,None -0aad0176e35b3749,BSI-DSZ-CC-0477-2007,None,None,None -0b0648a9b9a26747,BSI-DSZ-CC-0223-2003,None,None,None -0bd651fcbb86f2a9,ANSSI-CC-2020/74,None,None,None -0bd651fcbb86f2a9,BSI-DSZ-CC-1136-2021,None,None,None -0c9e215997d96f44,BSI-DSZ-CC-0555-2009,None,None,None -0cd406e7c5013b74,BSI-DSZ-CC-0304-2006,None,None,None -0d4a95a8b52b787b,BSI-DSZ-CC-0829-2012,None,None,None -0da9d81c9f563df5,BSI-DSZ-CC-0382-2007,None,None,None -0ee132dec087b772,ANSSI-CC-2011/07,None,None,None -0ef56cba2362e463,ANSSI-CC-2015/42,None,None,None -0f1b31ac93459d1f,BSI-DSZ-CC-0720-2011,None,None,None -0f684159ad31f883,BSI-DSZ-CC-0555-2009,None,None,None -0fe0bb1054f49b7f,CCEVS-VR-04-0092,None,None,None -0ff82b1cc24b3105,BSI-DSZ-CC-0302-2005,None,None,None -10233188161519ee,ANSSI-CC-2019/48,None,None,None -10233188161519ee,ANSSI-CC-2020/26,None,None,None -10233188161519ee,BSI-DSZ-CC-1059-2018,None,None,None -10233188161519ee,BSI-DSZ-CC-1059-V3-2019,None,None,None -102707a8c4d8a1b0,ANSSI-CC-2019/35,None,None,None -102707a8c4d8a1b0,ANSSI-CC-2020/26,None,None,None -102707a8c4d8a1b0,BSI-DSZ-CC-1059-2018,None,None,None -102707a8c4d8a1b0,BSI-DSZ-CC-1059-V3-2019,None,None,None -102bbdfcd696d7a9,ANSSI-CC-2018/13,None,None,None -102bbdfcd696d7a9,ANSSI-CC-2019/01,None,None,None -107c348f4ba0a41c,BSI-DSZ-CC-1135-2020,None,None,None -109eb2158ca6a2f9,BSI-DSZ-CC-0410-2007,None,None,None -10af4a6378d0fb7a,ANSSI-CC-2014/07,None,None,None -10af4a6378d0fb7a,ANSSI-CC-2014/86,None,None,None -10af4a6378d0fb7a,BSI-DSZ-CC-0837-V2-2014,None,None,None -10af4a6378d0fb7a,BSI-DSZ-CC-0845-V2-2013,None,None,None -10b17081dd7cad8f,ANSSI-CC-2015/36,None,None,None -10b17081dd7cad8f,ANSSI-CC-2017/39,None,None,None -112fe9f123e277ee,ANSSI-CC-2010/02,None,None,None -112fe9f123e277ee,ANSSI-CC-2010/19,None,None,None -114e39b4da949e6d,ANSSI-CC-2020/08,None,None,None -114e39b4da949e6d,BSI-DSZ-CC-1110-V2-2019,None,None,None -1172a88e009dc248,CRP257,None,None,None -1172a88e009dc248,CRP281,None,None,None -1188aebd8681e4fe,BSI-DSZ-CC-0465-2008,None,None,None -11a98c5d5ff6f40f,BSI-DSZ-CC-0403-2008,None,None,None -11a98c5d5ff6f40f,BSI-DSZ-CC-0468-2007,None,None,None -11a98c5d5ff6f40f,CRP244,None,None,None -122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None,None -123c94418ddf1ad8,BSI-DSZ-CC-0367-2006,None,None,None -131ec924c42ef90b,ANSSI-CC-2017/61,None,None,None -135ca3602f0005bf,BSI-DSZ-CC-0897-V2-2014,None,None,None -135ca3602f0005bf,NSCIB-CC-16-99111,None,None,None -13e5b88ad3becb77,BSI-DSZ-CC-0322-2005,None,None,None -13e5b88ad3becb77,BSI-DSZ-CC-0344-2005,None,None,None -1468948b4f0509ab,BSI-DSZ-CC-0813-2012,None,None,None -1494cff85b3dc2bb,BSI-DSZ-CC-0633-2010,None,None,None -157b741a0f529a0a,ANSSI-CC-2009/37,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0555-2009,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0633-2010,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0675-2011,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0745-2012,None,None,None -158950ed4bc35274,ANSSI-CC-2015/36,None,None,None -158950ed4bc35274,ANSSI-CC-2015/80,None,None,None -158950ed4bc35274,ANSSI-CC-2016/33,None,None,None -15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,None,None,None -15b121492722bffb,BSI-DSZ-CC-0754-2012,None,None,None -16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,None,None,None -16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,None,None,None -16b24ff1bf3c079b,ANSSI-CC-2012/22,None,None,None -173097995b7a7f12,ANSSI-CC-2012/49,None,None,None -173097995b7a7f12,ANSSI-CC-2013/28,None,None,None -173704f0d2b8a02f,ANSSI-CC-2018/08,None,None,None -173704f0d2b8a02f,ANSSI-CC-2018/19,None,None,None -177dee4c1051e612,BSI-DSZ-CC-0854-2013,None,None,None -17bfc3d0570ab3d0,ANSSI-CC-2016/09,None,None,None -183a5cd3e2aaa0ec,ANSSI-CC-2017/76,None,None,None -183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,None,None,None -18a515a0fff0c0c3,BSI-DSZ-CC-0957-V2-2016,None,None,None -18b7e5bdf459ca13,NSCIB-CC-07-09219,None,None,None -195366bfba7213a5,BSI-DSZ-CC-0266-2005,None,None,None -195366bfba7213a5,BSI-DSZ-CC-0322-2005,None,None,None -195366bfba7213a5,BSI-DSZ-CC-0399-2007,None,None,None -1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,None,None,None -19d3185c40be008d,ANSSI-CC-2016/65,None,None,None -1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,None,None,None -1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,None,None,None -1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,None,None,None -1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,None,None,None -1cc05dbb992431b9,ANSSI-CC-2021/29,None,None,None -1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,None,None,None -1d9ae732c5dec242,ANSSI-CC-2009/37,None,None,None -1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,None,None,None -1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,None,None,None -1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,None,None,None -1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,None,None,None -2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,None,None,None -21951e191e55b66e,BSI-DSZ-CC-0782-2012,None,None,None -21cffadbbb87c205,BSI-DSZ-CC-0348-2006,None,None,None -22ede463dbf1a105,ANSSI-CC-2015/36,None,None,None -22ede463dbf1a105,ANSSI-CC-2015/80,None,None,None -22ede463dbf1a105,ANSSI-CC-2016/43,None,None,None -22ede463dbf1a105,ANSSI-CC-2016/44,None,None,None -23a85a7f9c07412f,ANSSI-CC-2014/46,None,None,None -23a85a7f9c07412f,ANSSI-CC-2016/33,None,None,None -24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,None,None,None -24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,None,None,None -259df98e7476a843,BSI-DSZ-CC-0666-2012,None,None,None -259e0ab1c4b690cf,ANSSI-CC-2012/70,None,None,None -25c07c5d0a05d86c,ANSSI-CC-2015/08,None,None,None -25c07c5d0a05d86c,ANSSI-CC-2015/15,None,None,None -25c07c5d0a05d86c,BSI-DSZ-CC-0845-2012,None,None,None -25c07c5d0a05d86c,BSI-DSZ-CC-0845-V2-2013,None,None,None -26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,None,None,None -26c7b173668167d8,BSI-DSZ-CC-0813-2012,None,None,None -2760aeedce0b79db,ANSSI-CC-2011/64,None,None,None -2760aeedce0b79db,BSI-DSZ-CC-0555-2009,None,None,None -27ccec0740bb7915,ANSSI-CC-2010/02,None,None,None -27ccec0740bb7915,ANSSI-CC-2010/20,None,None,None -27ccec0740bb7915,ANSSI-CC-2010/35,None,None,None -296618f10b019d9c,BSI-DSZ-CC-0417-2008,None,None,None -29cc495628e76ffa,BSI-DSZ-CC-0845-2012,None,None,None -29cc495628e76ffa,BSI-DSZ-CC-0886-2013,None,None,None -29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,None,None,None -29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,None,None,None -2a07b85f61ac08b0,ANSSI-CC-2019/02,None,None,None -2a07b85f61ac08b0,ANSSI-CC-2019/59,None,None,None -2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,None,None,None -2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,None,None,None -2b0342d5b70b7a78,ANSSI-CC-2010/03,None,None,None -2bd9661657e78278,ANSSI-CC-2012/39,None,None,None -2bd9661657e78278,ANSSI-CC-2013/13,None,None,None -2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,None,None,None -2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,None,None,None -2d046178aa118fff,BSI-DSZ-CC-0293-2005,None,None,None -2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,None,None,None -2eee0fdd5cdaf565,ANSSI-CC-2017/07,None,None,None -2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,None,None,None -2f9feaf4121720da,BSI-DSZ-CC-0547-2009,None,None,None -30c6777901ae346c,BSI-DSZ-CC-0438-2007,None,None,None -30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,None,None,None -3183669bf78db43b,ANSSI-CC-2015/45,None,None,None -3183669bf78db43b,ANSSI-CC-2016/64,None,None,None -3183669bf78db43b,BSI-DSZ-CC-0782-V2-2015,None,None,None -31fd07069aebf013,BSI-DSZ-CC-0466-2008,None,None,None -328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,None,None,None -328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,None,None,None -3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,None,None,None -3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,None,None,None -337ece90615ed69d,ANSSI-CC-2010/02,None,None,None -3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,None,None,None -33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,None,None,None -33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,None,None,None -34506b8fab6dd7c3,ANSSI-CC-2014/22,None,None,None -34ed76f0c32d9e93,ANSSI-CC-2012/49,None,None,None -34ed76f0c32d9e93,ANSSI-CC-2013/18,None,None,None -3519c14e4114d93d,BSI-DSZ-CC-0782-2012,None,None,None -35b9be2203d2b9e6,ANSSI-CC-2012/70,None,None,None -35b9be2203d2b9e6,BSI-DSZ-CC-0895-2014,None,None,None -35efe13fa9e93a68,CRP178,None,None,None -35efe13fa9e93a68,CRP182,None,None,None -36641ce7192b5f92,BSI-DSZ-CC-0977-2017,None,None,None -36df04ab9c978ab9,BSI-DSZ-CC-0624-2010,None,None,None -373e7b1bc9066563,BSI-DSZ-CC-1110-V2-2019,None,None,None -3746512864b941d4,ANSSI-CC-2016/15,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0404-2007,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0410-2007,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0555-2009,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0633-2010,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0674-2011,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0675-2011,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0709-2010,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0710-2010,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0730-2011,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0797-2012,None,None,None -37b94c08f8e25249,BSI-DSZ-CC-0799-2012,None,None,None -37c8d23c44b95833,ANSSI-CC-2017/16,None,None,None -37d28734245a3bb1,BSI-DSZ-CC-0257-2004,None,None,None -37e0d3cb098458d0,ANSSI-CC-2014/50,None,None,None -37e0d3cb098458d0,BSI-DSZ-CC-0999-2016,None,None,None -37fe6036c2ac932b,ANSSI-CC-2014/06,None,None,None -37fe6036c2ac932b,ANSSI-CC-2014/07,None,None,None -37fe6036c2ac932b,BSI-DSZ-CC-0829-2012,None,None,None -3817c0aca007989e,ANSSI-CC-2011/10,None,None,None -3817c0aca007989e,ANSSI-CC-2011/12,None,None,None -3817c0aca007989e,BSI-DSZ-CC-0523-2008,None,None,None -3817c0aca007989e,BSI-DSZ-CC-0626-2009,None,None,None -384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,None,None,None -384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,None,None,None -3976c9e492193315,BSI-DSZ-CC-1113-2021,None,None,None -39c07ebbca541145,BSI-DSZ-CC-0917-2014,None,None,None -3a3b0a9113835307,BSI-DSZ-CC-0402-2008,None,None,None -3a3b0a9113835307,BSI-DSZ-CC-0468-2007,None,None,None -3a3b0a9113835307,CCEVS-VR-07-0054,None,None,None -3a3b0a9113835307,CCEVS-VR-VID10271-2007,None,None,None -3a6a5f536bfbd4d4,ANSSI-CC-2009/59,None,None,None -3ba9f330ce93636b,BSI-DSZ-CC-0640-2010,None,None,None -3ba9f330ce93636b,BSI-DSZ-CC-0677-2010,None,None,None -3d149fe7c08bfc58,CRP256,None,None,None -3d756c83419bba28,ANSSI-CC-2011/07,None,None,None -3d756c83419bba28,ANSSI-CC-2011/77,None,None,None -3d8083b1e6c7b336,BSI-DSZ-CC-1136-V2-2022,None,None,None -3d8083b1e6c7b336,BSI-DSZ-CC-1136-V3-2022,None,None,None -3e08a27e9d9c9b1e,BSI-DSZ-CC-0973-2016,None,None,None -3e08a27e9d9c9b1e,NSCIB-CC-16-99111,None,None,None -3e182053b03f1faf,CCEVS-VR-06-0044,None,None,None -3e345d3c6002ad55,BSI-DSZ-CC-0915-2016,None,None,None -3ea02a62856c752a,ANSSI-CC-2012/70,None,None,None -3ea02a62856c752a,ANSSI-CC-2012/72,None,None,None -3ea02a62856c752a,BSI-DSZ-CC-0719-2011,None,None,None -3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,None,None,None -3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,None,None,None -3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,None,None,None -3f4b6e4f245f6fab,ANSSI-CC-2017/49,None,None,None -3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,None,None,None -3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,None,None,None -3feda0b8b5637540,NSCIB-CC-67206-CR5,None,None,None -4057f5dbe4ffa4cd,ANSSI-CC-2017/57,None,None,None -40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,None,None,None -40fa0a4cb5193977,ANSSI-CC-2018/27,None,None,None -40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,None,None,None -40fc6ad0aed92913,ANSSI-CC-2017/54,None,None,None -40fc6ad0aed92913,BSI-DSZ-CC-0891-V2-2016,None,None,None -41bd2924e9afced5,ANSSI-CC-2010/49,None,None,None -41bd2924e9afced5,ANSSI-CC-2010/50,None,None,None -420876ec1e5ba657,ANSSI-CC-2012/30,None,None,None -424f6d496746258b,ANSSI-CC-2019/12,None,None,None -424f6d496746258b,BSI-DSZ-CC-0995-2018,None,None,None -4285d9b580f8a2a6,ANSSI-CC-2013/59,None,None,None -42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,None,None,None -42d2e68c29d9eef7,ANSSI-CC-2018/30,None,None,None -42f53e51476f1a3c,ANSSI-CC-2011/07,None,None,None -438b59086f6ebd64,BSI-DSZ-CC-1107-2020,None,None,None -43a4ca62d0c0b0da,ANSSI-CC-2017/63,None,None,None -43a4ca62d0c0b0da,ANSSI-CC-2017/64,None,None,None -43a4ca62d0c0b0da,BSI-DSZ-CC-1059-V3-2019,None,None,None -43d9cb9d3fe30ff2,BSI-DSZ-CC-0555-2009,None,None,None -43d9cb9d3fe30ff2,BSI-DSZ-CC-0857-2013,None,None,None -449c74a92ebb61a4,ANSSI-CC-2017/24,None,None,None -44becd7e128f4ba0,BSI-DSZ-CC-0410-2007,None,None,None -45098872448f5816,NSCIB-CC-0441513-CR,None,None,None -46459cfb9c1045d8,ANSSI-CC-2018/27,None,None,None -46459cfb9c1045d8,BSI-DSZ-CC-0782-V4-2018,None,None,None -4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,None,None,None -47913a485c3c8a18,ANSSI-CC-2019/28,None,None,None -4832a44c0df0bad2,ANSSI-CC-2019/12,None,None,None -4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,None,None,None -4893140ae5daadaf,BSI-DSZ-CC-0782-2012,None,None,None -4893140ae5daadaf,CRP278,None,None,None -48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,None,None,None -48d897f17754c7ce,BSI-DSZ-CC-0978-2016,None,None,None -48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,None,None,None -491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,None,None,None -491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,None,None,None -49e6665ee56d3bfc,ANSSI-CC-2015/45,None,None,None -49e6665ee56d3bfc,ANSSI-CC-2016/64,None,None,None -49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,None,None,None -4abc83fee59586b2,ANSSI-CC-2020/07,None,None,None -4abc83fee59586b2,BSI-DSZ-CC-1110-V3-2020,None,None,None -4abc83fee59586b2,NSCIB-CC-200270-CR,None,None,None -4b2f963b48e0f954,ANSSI-CC-2010/11,None,None,None -4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,None,None,None -4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,None,None,None -4bf7d6ef245dd393,ANSSI-CC-2009/60,None,None,None -4bf7d6ef245dd393,ANSSI-CC-2010/02,None,None,None -4c0e2794cb729825,ANSSI-CC-2010/02,None,None,None -4c0e2794cb729825,ANSSI-CC-2010/22,None,None,None -4c90a96fb1ee09f6,BSI-DSZ-CC-0410-2007,None,None,None -4d21561351ecd00a,BSI-DSZ-CC-1052-V3-2021,None,None,None -4d2a177384b23fd6,ANSSI-CC-2016/58,None,None,None -4d885e54f1361d04,BSI-DSZ-CC-0577-2009,None,None,None -4dbb108406acb1a5,ANSSI-CC-2010/02,None,None,None -4dbb108406acb1a5,ANSSI-CC-2012/68,None,None,None -4dc023ea2e3c4115,ANSSI-CC-2017/59,None,None,None -4dc023ea2e3c4115,BSI-DSZ-CC-0782-V3-2017,None,None,None -4dd0daec78093d31,ANSSI-CC-2016/59,None,None,None -4e29916c366fe997,BSI-DSZ-CC-0782-2012,None,None,None -4f3bf0c433b7f54f,BSI-DSZ-CC-0171-2002,None,None,None -4f8db13958be0f75,BSI-DSZ-CC-1051-2019,None,None,None -4f953ae00a24eb85,ANSSI-CC-2015/36,None,None,None -4f953ae00a24eb85,ANSSI-CC-2016/43,None,None,None -4f953ae00a24eb85,ANSSI-CC-2016/44,None,None,None -4f953ae00a24eb85,ANSSI-CC-2016/78,None,None,None -5103648897ba1b3a,BSI-DSZ-CC-0891-2015,None,None,None -5162627872566ada,ANSSI-CC-2020/07,None,None,None -5162627872566ada,BSI-DSZ-CC-1110-V2-2019,None,None,None -51768783ab3e67f3,ANSSI-CC-2010/03,None,None,None -51768783ab3e67f3,ANSSI-CC-2017/47,None,None,None -51768783ab3e67f3,ANSSI-CC-2017/49,None,None,None -51768783ab3e67f3,BSI-DSZ-CC-0955-V2-2016,None,None,None -51768783ab3e67f3,BSI-DSZ-CC-0973-V2-2016,None,None,None -51a4bea9a77a55cf,BSI-DSZ-CC-0410-2007,None,None,None -51a4bea9a77a55cf,BSI-DSZ-CC-0417-2008,None,None,None -51a4bea9a77a55cf,BSI-DSZ-CC-0602-2009,None,None,None -51c589803c349e44,BSI-DSZ-CC-0813-2012,None,None,None -51c589803c349e44,BSI-DSZ-CC-0851-2013,None,None,None -5346d5a4418466ab,BSI-DSZ-CC-1110-V3-2020,None,None,None -53bba35f7ea6edc9,ANSSI-CC-2017/02,None,None,None -53fe111411edfa45,BSI-DSZ-CC-0325-2006,None,None,None -541da571bac5e279,BSI-DSZ-CC-0466-2008,None,None,None -5423483cd3c3b75a,ANSSI-CC-2016/60,None,None,None -5423483cd3c3b75a,ANSSI-CC-2016/69,None,None,None -5423483cd3c3b75a,BSI-DSZ-CC-0946-V2-2015,None,None,None -5498bfcecf19dfa3,ANSSI-CC-2021/08,None,None,None -54f754dc95137c47,NSCIB-CC-13-37760-CR2,None,None,None -54fada9909a1edb3,ANSSI-CC-2009/56,None,None,None -54fada9909a1edb3,BSI-DSZ-CC-0451-2007,None,None,None -5506b5232076d409,CCEVS-VR-VID10334-2010,None,None,None -55669bcb3a9e09c0,BSI-DSZ-CC-1041-2018,None,None,None -55925bc39df20912,BSI-DSZ-CC-0322-2005,None,None,None -559e0907fef71cf6,ANSSI-CC-2014/62,None,None,None -559e0907fef71cf6,BSI-DSZ-CC-0829-2012,None,None,None -559e0907fef71cf6,BSI-DSZ-CC-0829-V2-2015,None,None,None -55e52bcaab69d573,BSI-DSZ-CC-0410-2007,None,None,None -55e52bcaab69d573,BSI-DSZ-CC-0463-2008,None,None,None -55e85aac6594c2fc,BSI-DSZ-CC-0410-2007,None,None,None -55e85aac6594c2fc,BSI-DSZ-CC-0417-2008,None,None,None -55e85aac6594c2fc,BSI-DSZ-CC-0643-2010,None,None,None -55f9f2fedfc7f069,BSI-DSZ-CC-0926-2014,None,None,None -568c473e6e5e95c7,BSI-DSZ-CC-0897-V2-2014,None,None,None -56a56d39dbfe3516,ANSSI-CC-2017/47,None,None,None -56a56d39dbfe3516,ANSSI-CC-2017/49,None,None,None -56a56d39dbfe3516,BSI-DSZ-CC-0973-V2-2016,None,None,None -56bca69ce012d4c1,BSI-DSZ-CC-0951-2015,None,None,None -56bca69ce012d4c1,BSI-DSZ-CC-1028-2017,None,None,None -56ceb1a9c003ac3f,BSI-DSZ-CC-1124-2021,None,None,None -56ebfd7a918891c6,2016-36-INF-1779,None,None,None -5787ff8207fd751d,ANSSI-CC-2021/51,None,None,None -57e108dcd6ee072b,BSI-DSZ-CC-0720-2011,None,None,None -57e929939f1fd1c0,BSI-DSZ-CC-0487-2009,None,None,None -583c12b5bf2423c6,BSI-DSZ-CC-1110-V4-2021,None,None,None -58688dfc2c59d3ab,BSI-DSZ-CC-0831-V2-2020,None,None,None -58688dfc2c59d3ab,BSI-DSZ-CC-0957-V2-2016,None,None,None -587055293e9fa51d,ANSSI-CC-2017/07,None,None,None -587055293e9fa51d,BSI-DSZ-CC-0891-V2-2016,None,None,None -58ece75f7f2b5099,ANSSI-CC-2010/02,None,None,None -58ece75f7f2b5099,ANSSI-CC-2012/68,None,None,None -59820117bb7fa240,ANSSI-CC-2009/61,None,None,None -59820117bb7fa240,ANSSI-CC-2010/02,None,None,None -59820117bb7fa240,ANSSI-CC-2010/31,None,None,None -598e2a9978d3c79c,ANSSI-CC-2013/59,None,None,None -59b026d889aa0309,BSI-DSZ-CC-1025-V2-2019,None,None,None -5a1764ef77908457,BSI-DSZ-CC-1040-2019,None,None,None -5b1b55f1997a9439,OCSI/CERT/IMQ/04/2016/RC,None,None,None -5b67a83de2aaa93a,ANSSI-CC-2013/02,None,None,None -5b67a83de2aaa93a,ANSSI-CC-2014/37,None,None,None -5bf118ffa29520c0,ANSSI-CC-2017/76,None,None,None -5bf118ffa29520c0,BSI-DSZ-CC-0891-V2-2016,None,None,None -5bfbc4a6cbab9eb1,BSI-DSZ-CC-0302-2005,None,None,None -5c5431cca026fb3e,BSI-DSZ-CC-0179-2003,None,None,None -5c5431cca026fb3e,BSI-DSZ-CC-0213-2003,None,None,None -5caf1c90ec7001b7,BSI-DSZ-CC-0410-2007,None,None,None -5caf1c90ec7001b7,BSI-DSZ-CC-0417-2008,None,None,None -5cbd46653c3bea1a,BSI-DSZ-CC-0439-2008,None,None,None -5cec11774890d61e,BSI-DSZ-CC-0750-V2-2014,None,None,None -5cec11774890d61e,BSI-DSZ-CC-0858-2013,None,None,None -5cec11774890d61e,BSI-DSZ-CC-0911-2014,None,None,None -5cec11774890d61e,BSI-DSZ-CC-0912-2014,None,None,None -5cec11774890d61e,BSI-DSZ-CC-0914-2014,None,None,None -5cec11774890d61e,NSCIB-CC-13-37760-CR2,None,None,None -5cfe68ed449e0478,ANSSI-CC-2017/76,None,None,None -5cfe68ed449e0478,BSI-DSZ-CC-0891-V2-2016,None,None,None -5d4aae047020163b,ANSSI-CC-2010/03,None,None,None -5d4aae047020163b,ANSSI-CC-2017/47,None,None,None -5d4aae047020163b,ANSSI-CC-2017/49,None,None,None -5d4aae047020163b,BSI-DSZ-CC-0955-V2-2016,None,None,None -5d4aae047020163b,BSI-DSZ-CC-0973-V2-2016,None,None,None -5d674c06a559a5e7,BSI-DSZ-CC-1128-V4-2022,None,None,None -5e4e16da9d0630dd,ANSSI-CC-2016/70,None,None,None -5e4e16da9d0630dd,ANSSI-CC-2017/59,None,None,None -5eb91b620bbf5bfc,ANSSI-CC-2009/39,None,None,None -5eb91b620bbf5bfc,ANSSI-CC-2009/40,None,None,None -5eda77c801e2df86,BSI-DSZ-CC-0891-V4-2019,None,None,None -5edefd19948ad581,CRP237,None,None,None -5ee9898421c06859,BSI-DSZ-CC-0758-2012,None,None,None -5efe98a1ba4df4d7,BSI-DSZ-CC-0757-2011,None,None,None -5f259ce91f0234a0,BSI-DSZ-CC-1110-V3-2020,None,None,None -5f4cd1504f8b18e3,CCEVS-VR-05-0131,None,None,None -5f84cd800ee96468,BSI-DSZ-CC-0750-V2-2014,None,None,None -5f84cd800ee96468,NSCIB-CC-13-37760-CR2,None,None,None -5f99b00bf258f32f,ANSSI-CC-2010/03,None,None,None -5f99b00bf258f32f,ANSSI-CC-2017/47,None,None,None -5f99b00bf258f32f,ANSSI-CC-2017/49,None,None,None -5f99b00bf258f32f,BSI-DSZ-CC-0955-V2-2016,None,None,None -5f99b00bf258f32f,BSI-DSZ-CC-0973-V2-2016,None,None,None -60ec8738947badb0,ANSSI-CC-2017/12,None,None,None -60ec8738947badb0,ANSSI-CC-2018/12,None,None,None -60f08c85a518de1c,ANSSI-CC-2009/61,None,None,None -60f08c85a518de1c,ANSSI-CC-2010/02,None,None,None -60f0dd83c8f32b8c,ANSSI-CC-2014/46,None,None,None -6136acacc653106c,BSI-DSZ-CC-0555-2009,None,None,None -61e862ed07d3cb03,BSI-DSZ-CC-0875-2015,None,None,None -62313881916550ca,ANSSI-CC-2019/28,None,None,None -62eb991153211de6,ANSSI-CC-2010/02,None,None,None -62eb991153211de6,BSI-DSZ-CC-0671-2011,None,None,None -6313a92bf426d2bb,BSI-DSZ-CC-0411-2007,None,None,None -6337d3dcc5362dc6,ANSSI-CC-2019/12,None,None,None -637657613004dbbd,BSI-DSZ-CC-0293-2005,None,None,None -63e55780fa1766de,ANSSI-CC-2010/01,None,None,None -64e80c255d51ff37,BSI-DSZ-CC-0404-2007,None,None,None -64e80c255d51ff37,BSI-DSZ-CC-0410-2007,None,None,None -64e80c255d51ff37,BSI-DSZ-CC-0411-2007,None,None,None -64ed7e55bdc50177,ANSSI-CC-2020/07,None,None,None -64ed7e55bdc50177,BSI-DSZ-CC-1110-V2-2019,None,None,None -6529877d9799822a,BSI-DSZ-CC-1113-V3-2022,None,None,None -654554952ecb20ac,ANSSI-CC-2010/01,None,None,None -6587b26878da24b5,ANSSI-CC-2015/08,None,None,None -6587b26878da24b5,ANSSI-CC-2015/15,None,None,None -6587b26878da24b5,ANSSI-CC-2015/30,None,None,None -6587b26878da24b5,BSI-DSZ-CC-0845-2012,None,None,None -6587b26878da24b5,BSI-DSZ-CC-0870-2014,None,None,None -65a47f93451f13de,BSI-DSZ-CC-1079-V2-2020,None,None,None -65af80e5c6538215,BSI-DSZ-CC-1059-V2-2019,None,None,None -65fba51313f65938,ANSSI-CC-2015/66,None,None,None -6700028a309c3f7f,CRP271,None,None,None -6700028a309c3f7f,CRP282,None,None,None -67240adc18be338d,BSI-DSZ-CC-1044-V5-2022,None,None,None -67e75c6c6538110e,BSI-DSZ-CC-0963-2015,None,None,None -68875c05e00bd1aa,ANSSI-CC-2009/26,None,None,None -692e91451741ef49,BSI-DSZ-CC-0268-2005,None,None,None -693325ca20a6325b,BSI-DSZ-CC-0917-2014,None,None,None -696c193b3d13924e,OCSI/CERT/RES/03/2012/RC,None,None,None -6999db84bd629b32,BSI-DSZ-CC-0293-2005,None,None,None -6999db84bd629b32,BSI-DSZ-CC-0296-2006,None,None,None -69b877d99c360f97,BSI-DSZ-CC-0837-V2-2014,None,None,None -6a46ff1d1a5154b0,BSI-DSZ-CC-0845-2012,None,None,None -6abef2102c86c0fc,ANSSI-CC-2011/07,None,None,None -6ae356d8abbfdeb9,BSI-DSZ-CC-1153-2021,None,None,None -6b605ff743c859d0,ANSSI-CC-2015/36,None,None,None -6b605ff743c859d0,ANSSI-CC-2017/02,None,None,None -6b605ff743c859d0,ANSSI-CC-2018/41,None,None,None -6bdca9f13e19572e,ANSSI-CC-2019/34,None,None,None -6bdca9f13e19572e,ANSSI-CC-2020/26,None,None,None -6bdca9f13e19572e,BSI-DSZ-CC-1059-2018,None,None,None -6bdca9f13e19572e,BSI-DSZ-CC-1059-V3-2019,None,None,None -6c2fc8308efeccb4,BSI-DSZ-CC-0555-2009,None,None,None -6c2fc8308efeccb4,BSI-DSZ-CC-0633-2010,None,None,None -6c2fc8308efeccb4,BSI-DSZ-CC-0675-2011,None,None,None -6c3e0fe95400f1f5,BSI-DSZ-CC-0939-V2-2016,None,None,None -6cef8442092da1da,ANSSI-CC-2021/52,None,None,None -6d9c2bab8cc8d28c,ANSSI-CC-2014/06,None,None,None -6d9c2bab8cc8d28c,ANSSI-CC-2014/14,None,None,None -6d9c2bab8cc8d28c,ANSSI-CC-2014/49,None,None,None -6d9c2bab8cc8d28c,BSI-DSZ-CC-0695-2011,None,None,None -6d9c2bab8cc8d28c,BSI-DSZ-CC-0829-2012,None,None,None -6dde54b8c6d592ef,BSI-DSZ-CC-1086-2018,None,None,None -6df8f86449e56c05,ANSSI-CC-2017/24,None,None,None -6df8f86449e56c05,ANSSI-CC-2018/32,None,None,None -6e9496b7254e02e2,ANSSI-CC-2020/08,None,None,None -6e9496b7254e02e2,BSI-DSZ-CC-1110-V2-2019,None,None,None -6eacef3b2f976ce9,BSI-DSZ-CC-0814-2012,None,None,None -6eeed6fefb1f0243,ANSSI-CC-2019/28,None,None,None -6ef82dead8e3437f,BSI-DSZ-CC-0837-2013,None,None,None -6ef82dead8e3437f,BSI-DSZ-CC-0892-2014,None,None,None -6ef82dead8e3437f,NSCIB-CC-12-36243,None,None,None -6f13e8b708b43383,ANSSI-CC-2009/28,None,None,None -6f480809fc73b635,BSI-DSZ-CC-0978-V2-2017,None,None,None -6f90b0bf6bbe884b,BSI-DSZ-CC-0916-2015,None,None,None -6f90b0bf6bbe884b,BSI-DSZ-CC-0950-2017,None,None,None -6fd73557577cf9ed,ANSSI-CC-2020/21,None,None,None -6fe5828936663aed,BSI-DSZ-CC-0482-2008,None,None,None -6ff89f3123a6a98f,BSI-DSZ-CC-0782-V3-2017,None,None,None -70908a4cfdc5585c,BSI-DSZ-CC-0266-2005,None,None,None -70908a4cfdc5585c,BSI-DSZ-CC-0437-2008,None,None,None -7096b882b315807a,BSI-DSZ-CC-0955-V2-2016,None,None,None -70f75a14390522b6,BSI-DSZ-CC-0837-V2-2014,None,None,None -711a7b593ed41b02,ANSSI-CC-2010/49,None,None,None -71d3720cad53b289,BSI-DSZ-CC-0891-V4-2019,None,None,None -71d3720cad53b289,BSI-DSZ-CC-1105-2020,None,None,None -71eae9be60b7fb6c,BSI-DSZ-CC-0555-2009,None,None,None -71eae9be60b7fb6c,BSI-DSZ-CC-0633-2010,None,None,None -723ba544431f7874,BSI-DSZ-CC-1110-V4-2021,None,None,None -731aac44217611ec,ANSSI-CC-2014/24,None,None,None -736ede0289146b59,BSI-DSZ-CC-1067-2020,None,None,None -7386d04b20f354f4,BSI-DSZ-CC-0829-2012,None,None,None -7386d04b20f354f4,BSI-DSZ-CC-0829-V2-2015,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0404-2007,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0410-2007,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0555-2009,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0633-2010,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0674-2011,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0675-2011,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0709-2010,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0710-2010,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0730-2011,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0797-2012,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0798-2012,None,None,None -743318f3a2b4e281,BSI-DSZ-CC-0799-2012,None,None,None -74d685c3723bdf92,BSI-DSZ-CC-0978-2016,None,None,None -75ce95f62fc65fac,BSI-DSZ-CC-0269-2006,None,None,None -75ce95f62fc65fac,BSI-DSZ-CC-0376-2006,None,None,None -76d52f916b4222b1,BSI-DSZ-CC-0266-2005,None,None,None -76d52f916b4222b1,BSI-DSZ-CC-0470-2008,None,None,None -781c243d0021d4c5,BSI-DSZ-CC-0223-2003,None,None,None -78a279b07a6d3367,CCEVS-VR-05-0119,None,None,None -78ef46252d580af0,ANSSI-CC-2016/22,None,None,None -793075287bf70a81,BSI-DSZ-CC-1139-V2-2021,None,None,None -7942e835fb18c4ac,BSI-DSZ-CC-0482-2008,None,None,None -797ea41cc2e95f08,ANSSI-CC-2012/22,None,None,None -797ea41cc2e95f08,ANSSI-CC-2013/36,None,None,None -7980d204dbc3ecee,BSI-DSZ-CC-1113-2021,None,None,None -7a059d2754e088a3,ANSSI-CC-2022/10,None,None,None -7a059d2754e088a3,ANSSI-CC-2022/13,None,None,None -7a059d2754e088a3,BSI-DSZ-CC-1107-V2-2021,None,None,None -7a26d92281df3e8e,BSI-DSZ-CC-0451-2007,None,None,None -7a5a4a9578503a55,BSI-DSZ-CC-1072-V3-2019,None,None,None -7b63c31cfd0eb65f,BSI-DSZ-CC-0977-V2-2019,None,None,None -7b63c31cfd0eb65f,BSI-DSZ-CC-1019-V2-2019,None,None,None -7c04881c9a46b337,NSCIB-CC-09-11025,None,None,None -7c708f62e3280ec3,BSI-DSZ-CC-0354-2006,None,None,None -7c7e48e3aed8bfd4,ANSSI-CC-2020/08,None,None,None -7c7e48e3aed8bfd4,BSI-DSZ-CC-1110-V2-2019,None,None,None -7d4585a4b5b6e873,ANSSI-CC-2010/02,None,None,None -7dfcefd1ffeaa941,ANSSI-CC-2018/12,None,None,None -7e24168630d5124e,ANSSI-CC-2021/02,None,None,None -7e37ff763a97bc72,BSI-DSZ-CC-0555-2009,None,None,None -7e37ff763a97bc72,BSI-DSZ-CC-0857-2013,None,None,None -7e8763f553e43d12,BSI-DSZ-CC-0782-2012,None,None,None -7e92dcac20e009a5,BSI-DSZ-CC-0827-V3-2015,None,None,None -7f02a580f5219377,BSI-DSZ-CC-1040-2019,None,None,None -8035f242b12cec6d,ANSSI-CC-2009/26,None,None,None -8035f242b12cec6d,ANSSI-CC-2009/28,None,None,None -8035f242b12cec6d,ANSSI-CC-2009/50,None,None,None -808de32aa819821d,BSI-DSZ-CC-1021-V2-2017,None,None,None -80fe3ba3d9c9bf64,BSI-DSZ-CC-0406-2007,None,None,None -80fe3ba3d9c9bf64,BSI-DSZ-CC-0410-2007,None,None,None -819d92a6492415d3,BSI-DSZ-CC-0728-2011,None,None,None -819d92a6492415d3,BSI-DSZ-CC-0763-2012,None,None,None -82a2bd8fe4ed2087,BSI-DSZ-CC-0645-2010,None,None,None -82a2bd8fe4ed2087,BSI-DSZ-CC-0731-2011,None,None,None -82a2bd8fe4ed2087,BSI-DSZ-CC-0743-2011,None,None,None -82a2bd8fe4ed2087,BSI-DSZ-CC-0750-2011,None,None,None -83020f91ce1aaf68,ANSSI-CC-2017/24,None,None,None -83020f91ce1aaf68,ANSSI-CC-2018/32,None,None,None -835211383d281d89,BSI-DSZ-CC-1079-V2-2020,None,None,None -83b4d7d35a747d1b,BSI-DSZ-CC-0829-2012,None,None,None -83bb8420c3cccf85,BSI-DSZ-CC-0464-2008,None,None,None -83c1ee8ab43ea00a,ANSSI-CC-2010/02,None,None,None -84444ea4fabcc66c,BSI-DSZ-CC-1107-V2-2021,None,None,None -84501b4439b66fd9,BSI-DSZ-CC-0837-V2-2014,None,None,None -84b1520706c3d784,NSCIB-CC-13-37760-CR2,None,None,None -84b1520706c3d784,OCSI/CERT/SYS/05/2015/RC,None,None,None -84c8bf1f809380bc,ANSSI-CC-2018/12,None,None,None -84ff17001564ea47,BSI-DSZ-CC-1040-2019,None,None,None -85fa17a19251a2ea,BSI-DSZ-CC-0410-2007,None,None,None -85fa17a19251a2ea,BSI-DSZ-CC-0709-2010,None,None,None -86462bcec4492bfa,ANSSI-CC-2017/73,None,None,None -86462bcec4492bfa,ANSSI-CC-2020/22,None,None,None -86e9da10bc176267,ANSSI-CC-2020/05,None,None,None -871066e9e2eaba52,ANSSI-CC-2022/39,None,None,None -8782c7c292ef2759,BSI-DSZ-CC-0782-V2-2015,None,None,None -8782c7c292ef2759,BSI-DSZ-CC-0879-V2-2015,None,None,None -878af76320373ba0,BSI-DSZ-CC-0874-2014,None,None,None -87abd998f3b7e26f,ANSSI-CC-2015/36,None,None,None -87abd998f3b7e26f,ANSSI-CC-2016/43,None,None,None -87abd998f3b7e26f,ANSSI-CC-2016/44,None,None,None -87abd998f3b7e26f,ANSSI-CC-2016/77,None,None,None -8817adf6eaf4f06c,OCSI/CERT/SYS/10/2016/RC,None,None,None -8820852f4a1163b3,BSI-DSZ-CC-1110-V2-2019,None,None,None -8825e2c99223c084,BSI-DSZ-CC-0427-2007,None,None,None -882a9d016c7e1f36,ANSSI-CC-2009/38,None,None,None -8865aed38c79d3fc,BSI-DSZ-CC-0606-2010,None,None,None -8870ddf70bf57890,ANSSI-CC-2012/70,None,None,None -8870ddf70bf57890,ANSSI-CC-2012/72,None,None,None -8944c4f14bbcaa74,ANSSI-CC-2020/94,None,None,None -89d1f7ab8b77e41d,BSI-DSZ-CC-0831-2018,None,None,None -89d1f7ab8b77e41d,BSI-DSZ-CC-0957-V2-2016,None,None,None -8a5f8686ab9de19e,BSI-DSZ-CC-0891-2015,None,None,None -8a7731e9eb363fb4,BSI-DSZ-CC-0410-2007,None,None,None -8a7731e9eb363fb4,BSI-DSZ-CC-0417-2008,None,None,None -8ade0910a89717a8,BSI-DSZ-CC-0977-2017,None,None,None -8ae9a7fbbb50b6ed,ANSSI-CC-2012/77,None,None,None -8b0204e041dd3eb8,ANSSI-CC-2019/12,None,None,None -8b7145c13f36c324,BSI-DSZ-CC-1110-V3-2020,None,None,None -8bbbd89d09bb82ab,BSI-DSZ-CC-0555-2009,None,None,None -8bbbd89d09bb82ab,BSI-DSZ-CC-0857-2013,None,None,None -8bc3658bc958f81f,BSI-DSZ-CC-0837-V2-2014,None,None,None -8c6c4b4606608dfe,ANSSI-CC-2012/08,None,None,None -8cfcf8dc64485a74,ANSSI-CC-2009/28,None,None,None -8cfd0c9f4bcd21b8,BSI-DSZ-CC-0707-2012,None,None,None -8cfd0c9f4bcd21b8,BSI-DSZ-CC-0864-2012,None,None,None -8d2c50e41a1a0061,ANSSI-CC-2012/77,None,None,None -8d68ee5c283ca6b0,BSI-DSZ-CC-0552-2009,None,None,None -8d92202f35f15712,ANSSI-CC-2013/27,None,None,None -8d92202f35f15712,ANSSI-CC-2014/56,None,None,None -8dcdbce3262bfcd6,ANSSI-CC-2014/58,None,None,None -8e713eb632acc712,BSI-DSZ-CC-0782-V4-2018,None,None,None -8ee7145a1b48b578,ANSSI-CC-2017/76,None,None,None -8ee7145a1b48b578,BSI-DSZ-CC-0891-V4-2019,None,None,None -8f7cc0f456186028,BSI-DSZ-CC-0203-2003,None,None,None -8f7cc0f456186028,BSI-DSZ-CC-0205-2003,None,None,None -8f7e22207cb3dd8b,ANSSI-CC-2014/06,None,None,None -8f7e22207cb3dd8b,BSI-DSZ-CC-0829-2012,None,None,None -8f81bb35e37f1b4c,ANSSI-CC-2021/29,None,None,None -8f81bb35e37f1b4c,BSI-DSZ-CC-1107-2020,None,None,None -902dbd430adb77a5,BSI-DSZ-CC-0869-V2-2019,None,None,None -902dbd430adb77a5,BSI-DSZ-CC-0891-V3-2018,None,None,None -915e919390e6f58b,ANSSI-CC-2015/59,None,None,None -921ecc7c3e5f4884,BSI-DSZ-CC-0309-2006,None,None,None -9231adbf747903df,ANSSI-CC-2012/27,None,None,None -925da3b468003e72,ANSSI-CC-2021/35-R01,None,None,None -925da3b468003e72,NSCIB-CC-0530096-CR,None,None,None -9292ab2134e8a712,BSI-DSZ-CC-0827-V7-2018,None,None,None -9292ab2134e8a712,BSI-DSZ-CC-1015-2017,None,None,None -9299f6cd8a266541,BSI-DSZ-CC-0404-2007,None,None,None -9299f6cd8a266541,BSI-DSZ-CC-0417-2008,None,None,None -92ae986997c1d45c,BSI-DSZ-CC-0977-2017,None,None,None -92ae986997c1d45c,BSI-DSZ-CC-1019-2017,None,None,None -92c7355f5ea28e52,CRP225,None,None,None -932cef2b9698dde8,BSI-DSZ-CC-0283-2006,None,None,None -9337ab9573b366c1,BSI-DSZ-CC-0329-2006,None,None,None -93f14cad1cb201fa,BSI-DSZ-CC-0827-V6-2017,None,None,None -940a18f62c79ffae,ANSSI-CC-2021/51,None,None,None -953e132dd43ec03f,BSI-DSZ-CC-1058-2018,None,None,None -9545f2b0b8b92c8d,BSI-DSZ-CC-0410-2007,None,None,None -9545f2b0b8b92c8d,BSI-DSZ-CC-0527-2008,None,None,None -95774db1eb19b767,ANSSI-CC-2018/27,None,None,None -95c52fa37ef1eb3b,BSI-DSZ-CC-0791-2012,None,None,None -96106c2d31eea8b2,BSI-DSZ-CC-1039-2017,None,None,None -966a03f9f111aa35,ANSSI-CC-2021/42,None,None,None -966a03f9f111aa35,BSI-DSZ-CC-1107-V2-2021,None,None,None -96b0895b9f712d2f,BSI-DSZ-CC-0410-2007,None,None,None -96b0895b9f712d2f,BSI-DSZ-CC-0518-2008,None,None,None -96c685a3cdd9ed59,BSI-DSZ-CC-0510-2008,None,None,None -96c904c81aa2a241,ANSSI-CC-2017/07,None,None,None -96c904c81aa2a241,BSI-DSZ-CC-0891-V2-2016,None,None,None -975dae6e02765a90,ANSSI-CC-2009/63,None,None,None -977b04d04def60a7,ANSSI-CC-2019/28,None,None,None -97eb8171b51f9aad,BSI-DSZ-CC-0944-2014,None,None,None -97eb8171b51f9aad,BSI-DSZ-CC-0963-V2-2017,None,None,None -97f302708066dc04,BSI-DSZ-CC-0639-2010,None,None,None -981f7ea91d7bdcb6,ANSSI-CC-2014/06,None,None,None -981f7ea91d7bdcb6,BSI-DSZ-CC-0837-2013,None,None,None -983d431922822165,BSI-DSZ-CC-0837-2013,None,None,None -98642c6f99438194,ANSSI-CC-2020/04,None,None,None -988394e53c8e52d9,BSI-DSZ-CC-0976-V3-2019,None,None,None -988394e53c8e52d9,BSI-DSZ-CC-1110-V3-2020,None,None,None -98a578f7758bb518,BSI-DSZ-CC-0438-2007,None,None,None -9938be53082a02e6,NSCIB-CC-16-99111,None,None,None -994a2bf994b4facd,ANSSI-CC-2010/02,None,None,None -994a2bf994b4facd,ANSSI-CC-2012/68,None,None,None -994a2bf994b4facd,ANSSI-CC-2015/16,None,None,None -996691f0e03e303d,BSI-DSZ-CC-0555-2009,None,None,None -997281c99ac93b3b,ANSSI-CC-2010/03,None,None,None -997281c99ac93b3b,ANSSI-CC-2010/04,None,None,None -9973eb31c46f8637,ANSSI-CC-2010/08,None,None,None -9973eb31c46f8637,ANSSI-CC-2011/67,None,None,None -9ac6a8bda4fb7dc4,BSI-DSZ-CC-0751-2013,None,None,None -9ac6a8bda4fb7dc4,BSI-DSZ-CC-0813-2012,None,None,None -9ae7edbd0444982d,BSI-DSZ-CC-0837-V2-2014,None,None,None -9b30bcdbccfd4c68,BSI-DSZ-CC-0740-2011,None,None,None -9ba1fa1ae8c3ae67,ANSSI-CC-2016/65,None,None,None -9be0c86102117436,ANSSI-CC-2021/35-R01,None,None,None -9be76c10474e0c80,BSI-DSZ-CC-0758-2012,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0404-2007,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0410-2007,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0555-2009,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0633-2010,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0674-2011,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0675-2011,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0680-2010,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0709-2010,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0710-2010,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0730-2011,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0798-2012,None,None,None -9cc6c08cbaa694f2,BSI-DSZ-CC-0799-2012,None,None,None -9d36f8ab4e1b45dc,BSI-DSZ-CC-0324-2006,None,None,None -9e55246befe96fb4,ANSSI-CC-2010/01,None,None,None -9e7c513fdd35df17,ANSSI-CC-2010/40,None,None,None -9e7c513fdd35df17,ANSSI-CC-2011/64,None,None,None -9e7c513fdd35df17,BSI-DSZ-CC-0555-2009,None,None,None -9e7c513fdd35df17,BSI-DSZ-CC-0645-2010,None,None,None -9f400f89490fb8d5,BSI-DSZ-CC-0813-2012,None,None,None -9f8c9a3aa64a9f8e,ANSSI-CC-2019/01,None,None,None -a00e93f2981ef418,CRP292,None,None,None -a0fff0700364df49,BSI-DSZ-CC-0829-V2-2015,None,None,None -a0fff0700364df49,BSI-DSZ-CC-0952-2014,None,None,None -a14f221cf1761a8c,ANSSI-CC-2015/15,None,None,None -a14f221cf1761a8c,BSI-DSZ-CC-0845-V2-2013,None,None,None -a191b2da409f3518,BSI-DSZ-CC-0410-2007,None,None,None -a191b2da409f3518,BSI-DSZ-CC-0521-2008,None,None,None -a289b9d43310182b,BSI-DSZ-CC-0973-2016,None,None,None -a289b9d43310182b,BSI-DSZ-CC-0973-V2-2016,None,None,None -a29d46ab6efee468,BSI-DSZ-CC-0837-V2-2014,None,None,None -a2a5ccdf0846ce67,BSI-DSZ-CC-0460-2008,None,None,None -a2cbd066271b6b3b,BSI-DSZ-CC-0586-2009,None,None,None -a334142059d865a2,ANSSI-CC-2017/75,None,None,None -a334142059d865a2,ANSSI-CC-2018/61,None,None,None -a35ed74b7ea3ec4b,ANSSI-CC-2012/68,None,None,None -a3d152e25f8ce6c9,BSI-DSZ-CC-0870-2014,None,None,None -a465a767ceb7fd30,BSI-DSZ-CC-1015-2017,None,None,None -a49706f436a6873d,ANSSI-CC-2012/22,None,None,None -a49706f436a6873d,ANSSI-CC-2013/36,None,None,None -a5451ba38050a44d,BSI-DSZ-CC-0400-2007,None,None,None -a57845715134651a,BSI-DSZ-CC-1040-2019,None,None,None -a5898123ec124b34,ANSSI-CC-2015/36,None,None,None -a5898123ec124b34,ANSSI-CC-2015/80,None,None,None -a5898123ec124b34,ANSSI-CC-2016/43,None,None,None -a5898123ec124b34,ANSSI-CC-2016/44,None,None,None -a596f2412c0f8fe2,BSI-DSZ-CC-0808-2013,None,None,None -a596f2412c0f8fe2,BSI-DSZ-CC-0845-2012,None,None,None -a596f2412c0f8fe2,BSI-DSZ-CC-0885-2013,None,None,None -a5e8267322d0d7a6,ANSSI-CC-2010/40,None,None,None -a5e8267322d0d7a6,ANSSI-CC-2011/64,None,None,None -a5e8267322d0d7a6,BSI-DSZ-CC-0555-2009,None,None,None -a5e8267322d0d7a6,BSI-DSZ-CC-0645-2010,None,None,None -a64e6bf63ab6a299,ANSSI-CC-2013/32,None,None,None -a64e6bf63ab6a299,BSI-DSZ-CC-0829-2012,None,None,None -a67919286833a7df,BSI-DSZ-CC-0918-V2-2021,None,None,None -a67919286833a7df,BSI-DSZ-CC-0957-V2-2016,None,None,None -a6c1dd30884a197c,ANSSI-CC-2010/08,None,None,None -a76054a284f0ce61,ANSSI-CC-2014/86,None,None,None -a76054a284f0ce61,BSI-DSZ-CC-0837-V2-2014,None,None,None -a851c3b4377f4923,ANSSI-CC-2011/07,None,None,None -a9004216663480de,BSI-DSZ-CC-0782-V2-2015,None,None,None -a925d8f1adc5cfe2,BSI-DSZ-CC-0891-V3-2018,None,None,None -a925d8f1adc5cfe2,BSI-DSZ-CC-1001-2018,None,None,None -a925d8f1adc5cfe2,BSI-DSZ-CC-1033-2019,None,None,None -a925d8f1adc5cfe2,BSI-DSZ-CC-1034-2019,None,None,None -a93edea77d9f3338,BSI-DSZ-CC-0410-2007,None,None,None -a9f542d8e2be69fd,BSI-DSZ-CC-0782-2012,None,None,None -aa0213dd9f727219,BSI-DSZ-CC-0782-2012,None,None,None -aa0213dd9f727219,BSI-DSZ-CC-0879-2014,None,None,None -aa3f466daa34d3ff,BSI-DSZ-CC-0377-2007,None,None,None -aadbb9992a672a1b,ANSSI-CC-2019/01,None,None,None -ab769495f97aab9c,BSI-DSZ-CC-0879-V3-2018,None,None,None -ab88698a0c6ee1bb,BSI-DSZ-CC-0640-2010,None,None,None -ab8cf6af80a5c0e0,ANSSI-CC-2017/49,None,None,None -abef2b316c54f9e7,BSI-DSZ-CC-0978-2016,None,None,None -ac1257069b2f4afd,BSI-DSZ-CC-1113-V2-2021,None,None,None -ac4687f4b15507b6,ANSSI-CC-2014/58,None,None,None -ac4687f4b15507b6,ANSSI-CC-2014/59,None,None,None -acee5f619f09bf79,2015-27-INF-1539,None,None,None -ad2351479ff6ee84,ANSSI-CC-2009/61,None,None,None -ad2351479ff6ee84,ANSSI-CC-2010/02,None,None,None -ad2351479ff6ee84,ANSSI-CC-2010/29,None,None,None -adb678f7143014a0,ANSSI-CC-2015/15,None,None,None -adb678f7143014a0,BSI-DSZ-CC-0845-V2-2013,None,None,None -adc6971cdb0613b0,ANSSI-CC-2011/10,None,None,None -adc6971cdb0613b0,BSI-DSZ-CC-0523-2008,None,None,None -adc6971cdb0613b0,BSI-DSZ-CC-0626-2009,None,None,None -adc9df2489ce090a,BSI-DSZ-CC-0891-V4-2019,None,None,None -adc9df2489ce090a,BSI-DSZ-CC-1033-2019,None,None,None -adc9df2489ce090a,BSI-DSZ-CC-1034-V2-2021,None,None,None -adc9df2489ce090a,BSI-DSZ-CC-1105-2020,None,None,None -ae175aa839dbf692,NSCIB-CC-217812-CR3,None,None,None -ae9574428bd713e6,BSI-DSZ-CC-0891-V3-2018,None,None,None -ae9574428bd713e6,BSI-DSZ-CC-0995-2018,None,None,None -ae9574428bd713e6,BSI-DSZ-CC-1034-2019,None,None,None -aeb79c7e573de0e5,ANSSI-CC-2014/14,None,None,None -aeb79c7e573de0e5,ANSSI-CC-2014/86,None,None,None -aeb79c7e573de0e5,BSI-DSZ-CC-0837-2013,None,None,None -aeb79c7e573de0e5,BSI-DSZ-CC-0837-V2-2014,None,None,None -aeb79c7e573de0e5,BSI-DSZ-CC-0845-2012,None,None,None -af373587f2c57a24,BSI-DSZ-CC-0262-2005,None,None,None -af373587f2c57a24,CCEVS-VR-05-0131,None,None,None -af4c9b6e93062fde,ANSSI-CC-2010/03,None,None,None -af4c9b6e93062fde,ANSSI-CC-2017/47,None,None,None -af4c9b6e93062fde,ANSSI-CC-2017/49,None,None,None -af4c9b6e93062fde,BSI-DSZ-CC-0955-V2-2016,None,None,None -af4c9b6e93062fde,BSI-DSZ-CC-0973-V2-2016,None,None,None -af58ba642f4fc3d2,BSI-DSZ-CC-0891-V3-2018,None,None,None -af58ba642f4fc3d2,BSI-DSZ-CC-0980-2017,None,None,None -af78198822da03a7,ANSSI-CC-2010/03,None,None,None -afcf1165734847de,BSI-DSZ-CC-0207-2003,None,None,None -afe441f40a42865e,BSI-DSZ-CC-0902-2017,None,None,None -b0fd694500bfffab,BSI-DSZ-CC-0234-2004,None,None,None -b12b4e4eacad1497,BSI-DSZ-CC-0399-2007,None,None,None -b12b4e4eacad1497,BSI-DSZ-CC-0445-2007,None,None,None -b1401488f998c1f0,BSI-DSZ-CC-1136-2021,None,None,None -b15c099c68671895,ANSSI-CC-2017/49,None,None,None -b168db2651cbc1d6,ANSSI-CC-2010/02,None,None,None -b168db2651cbc1d6,ANSSI-CC-2012/69,None,None,None -b1b04d862f4c32af,BSI-DSZ-CC-0349-2006,None,None,None -b1b385d5bad724ed,BSI-DSZ-CC-1081-2020,None,None,None -b21be5dd6fdec83b,BSI-DSZ-CC-0965-2015,None,None,None -b2436dcf1c8e5dc5,BSI-DSZ-CC-0482-2008,None,None,None -b2436dcf1c8e5dc5,BSI-DSZ-CC-0528-2008,None,None,None -b24a14935edd51ad,BSI-DSZ-CC-0645-2010,None,None,None -b24a14935edd51ad,BSI-DSZ-CC-0750-2011,None,None,None -b252ff42ce0b7443,BSI-DSZ-CC-1040-2019,None,None,None -b2ba4d07e03b47ea,BSI-DSZ-CC-1178-V2-2022,None,None,None -b3855f3a6c06c01c,BSI-DSZ-CC-0645-2010,None,None,None -b4768c31e98b2170,BSI-DSZ-CC-0782-2012,None,None,None -b47b457f73c42c60,BSI-DSZ-CC-0918-2020,None,None,None -b47b457f73c42c60,BSI-DSZ-CC-0957-V2-2016,None,None,None -b49efd086851f84d,ANSSI-CC-2013/29,None,None,None -b49efd086851f84d,BSI-DSZ-CC-0555-2009,None,None,None -b49efd086851f84d,BSI-DSZ-CC-0645-2010,None,None,None -b4fc6a7b6c200b1b,BSI-DSZ-CC-1109-2019,None,None,None -b584e424a8b3dabe,BSI-DSZ-CC-0907-2013,None,None,None -b5968cd571a14cf3,BSI-DSZ-CC-0760-2013,None,None,None -b5968cd571a14cf3,BSI-DSZ-CC-0813-2012,None,None,None -b5985efd378af972,BSI-DSZ-CC-0897-V2-2014,None,None,None -b5b8bd0cb8bb7658,ANSSI-CC-2010/02,None,None,None -b5b8de155e4e40e6,ANSSI-CC-2015/36,None,None,None -b5b8de155e4e40e6,ANSSI-CC-2015/80,None,None,None -b644b78aafc367b0,BSI-DSZ-CC-0687-2011,None,None,None -b6a96b0dcf6eadf0,BSI-DSZ-CC-0955-V2-2016,None,None,None -b6c15035f759ec50,BSI-DSZ-CC-0269-2006,None,None,None -b6c3a9d53b9e784a,BSI-DSZ-CC-0750-V2-2014,None,None,None -b6c3a9d53b9e784a,NSCIB-CC-13-37760-CR2,None,None,None -b6e60b9025d116e4,ANSSI-CC-2014/46,None,None,None -b7132fe23fbdf9e1,BSI-DSZ-CC-0434-2007,None,None,None -b7132fe23fbdf9e1,BSI-DSZ-CC-0489-2008,None,None,None -b7209eec8fe7fc58,ANSSI-CC-2009/39,None,None,None -b764f5d5391e431e,BSI-DSZ-CC-0479-2008,None,None,None -b764f5d5391e431e,BSI-DSZ-CC-0482-2008,None,None,None -b804c318fc0c1e9b,BSI-DSZ-CC-0465-2008,None,None,None -b804c318fc0c1e9b,BSI-DSZ-CC-0550-2008,None,None,None -b82b48b0e915aaac,BSI-DSZ-CC-0169-2002,None,None,None -b883e389d34b1bb1,BSI-DSZ-CC-0410-2007,None,None,None -b883e389d34b1bb1,BSI-DSZ-CC-0417-2008,None,None,None -b883e389d34b1bb1,BSI-DSZ-CC-0521-2008,None,None,None -b887b7dcd6f1fdbb,BSI-DSZ-CC-0234-2004,None,None,None -b95f2b009d70fad1,BSI-DSZ-CC-0534-2009,None,None,None -b965c3b6a03aefc4,BSI-DSZ-CC-0348-2006,None,None,None -b99c572f58cfb19e,ANSSI-CC-2010/02,None,None,None -b99c572f58cfb19e,ANSSI-CC-2010/17,None,None,None -b99c572f58cfb19e,ANSSI-CC-2010/33,None,None,None -b9a6922899b66b8b,BSI-DSZ-CC-0813-2012,None,None,None -ba521df2c728c5a2,ANSSI-CC-2010/02,None,None,None -baead73ecc6e4a33,BSI-DSZ-CC-0421-2008,None,None,None -bb02df07a0d95e77,BSI-DSZ-CC-0322-2005,None,None,None -bb21cecb8bbfdf63,BSI-DSZ-CC-0269-2006,None,None,None -bb21cecb8bbfdf63,BSI-DSZ-CC-0376-2006,None,None,None -bb509db67c18bc40,BSI-DSZ-CC-0322-2005,None,None,None -bbc9a3a1d685d5f1,ANSSI-CC-2014/75,None,None,None -bc1452fff95744b9,BSI-DSZ-CC-0349-2006,None,None,None -bc25a76284e7c7ce,BSI-DSZ-CC-0782-2012,None,None,None -bc715a45d7584d95,BSI-DSZ-CC-0891-V2-2016,None,None,None -bd346dd3e46dff68,BSI-DSZ-CC-0639-2010,None,None,None -bdf8d5624e230986,BSI-DSZ-CC-0808-V2-2016,None,None,None -bdf8d5624e230986,BSI-DSZ-CC-0809-2013,None,None,None -bdf8d5624e230986,BSI-DSZ-CC-0978-2016,None,None,None -be9ca77bec616fe5,ANSSI-CC-2010/02,None,None,None -be9ca77bec616fe5,ANSSI-CC-2013/54,None,None,None -bec58e970e8cd6c8,BSI-DSZ-CC-0955-V2-2016,None,None,None -bf0f48ffd0dff903,BSI-DSZ-CC-0410-2007,None,None,None -bf0f48ffd0dff903,BSI-DSZ-CC-0463-2008,None,None,None -bf1338e9abe85c39,BSI-DSZ-CC-0410-2007,None,None,None -bff1330c13ccd99b,BSI-DSZ-CC-0293-2005,None,None,None -bff1330c13ccd99b,BSI-DSZ-CC-0346-2006,None,None,None -c06c7eb3afa4aea0,ANSSI-CC-2016/08,None,None,None -c10a3a62bf9fe782,BSI-DSZ-CC-1130-2021,None,None,None -c16f3dafab92df61,CCEVS-VR-05-0094,None,None,None -c1754dc2e7d666d8,BSI-DSZ-CC-0837-V2-2014,None,None,None -c18dc9f29967fe7f,BSI-DSZ-CC-0845-V2-2013,None,None,None -c1e14f36f031a342,BSI-DSZ-CC-1107-2020,None,None,None -c23218b044706800,BSI-DSZ-CC-0404-2007,None,None,None -c23218b044706800,BSI-DSZ-CC-0410-2007,None,None,None -c23218b044706800,BSI-DSZ-CC-0411-2007,None,None,None -c28289a45fc012a6,CRP210,None,None,None -c3dc8cfd97735115,BSI-DSZ-CC-0231-2004,None,None,None -c459c68e95d54f87,ANSSI-CC-2019/47,None,None,None -c459c68e95d54f87,ANSSI-CC-2020/26,None,None,None -c459c68e95d54f87,BSI-DSZ-CC-1059-2018,None,None,None -c459c68e95d54f87,BSI-DSZ-CC-1059-V3-2019,None,None,None -c483baec90d76587,BSI-DSZ-CC-0917-2014,None,None,None -c497da3fdb2f5027,ANSSI-CC-2017/49,None,None,None -c497da3fdb2f5027,BSI-DSZ-CC-1059-2018,None,None,None -c4ca6565698f0e49,BSI-DSZ-CC-1055-2018,None,None,None -c4e6a655a15144dd,BSI-DSZ-CC-0234-2004,None,None,None -c4f5a7748428e7d1,BSI-DSZ-CC-0349-2006,None,None,None -c518dd59d12daf17,BSI-DSZ-CC-0852-2013,None,None,None -c572cacbeb840621,BSI-DSZ-CC-0978-2016,None,None,None -c5bc8b961a199646,ANSSI-CC-2014/70,None,None,None -c621d1371b314913,BSI-DSZ-CC-0978-2016,None,None,None -c67724216ed5a318,BSI-DSZ-CC-0928-2018,None,None,None -c6c291cd525225b7,BSI-DSZ-CC-1025-2018,None,None,None -c8e381408b191a02,BSI-DSZ-CC-0410-2007,None,None,None -c8e381408b191a02,BSI-DSZ-CC-0417-2008,None,None,None -c8e381408b191a02,BSI-DSZ-CC-0575-2009,None,None,None -c8fc487eec95c21e,ANSSI-CC-2010/02,None,None,None -c902a788d699fe4a,SERTIT-003,None,None,None -c902a788d699fe4a,SERTIT-006,None,None,None -c96ac4c4015414ad,BSI-DSZ-CC-0782-V2-2015,None,None,None -ca1fa7c3f26bb3c5,BSI-DSZ-CC-0178-2003,None,None,None -ca63425a882729f3,BSI-DSZ-CC-0891-V4-2019,None,None,None -ca63425a882729f3,BSI-DSZ-CC-1033-V2-2021,None,None,None -ca63425a882729f3,BSI-DSZ-CC-1105-2020,None,None,None -ca88a34b0fcc5b0e,BSI-DSZ-CC-0280-2004,None,None,None -cb9a110799bc37f8,BSI-DSZ-CC-0954-2015,None,None,None -cbe5be4b63677c25,BSI-DSZ-CC-0752-2013,None,None,None -cc5a3d2a52f99ab3,ANSSI-CC-2018/25,None,None,None -cc5e2f58b681e555,BSI-DSZ-CC-0437-2008,None,None,None -cc5e2f58b681e555,BSI-DSZ-CC-0668-2010,None,None,None -cc64f3b8666b17bf,BSI-DSZ-CC-1072-2018,None,None,None -cc6918e95b803734,BSI-DSZ-CC-0891-V4-2019,None,None,None -cc6918e95b803734,BSI-DSZ-CC-1112-2020,None,None,None -cc932dbc293df39e,BSI-DSZ-CC-0973-2016,None,None,None -cc99b5d3806b1e97,ANSSI-CC-2016/66,None,None,None -cc99b5d3806b1e97,ANSSI-CC-2016/67,None,None,None -cc99b5d3806b1e97,BSI-DSZ-CC-0782-V2-2015,None,None,None -cc99b5d3806b1e97,BSI-DSZ-CC-0879-V2-2015,None,None,None -cce1a95634c6972b,BSI-DSZ-CC-0583-2009,None,None,None -cce214002aca738f,BSI-DSZ-CC-0897-V2-2014,None,None,None -cce214002aca738f,NSCIB-CC-16-99111,None,None,None -cd72fc2b62780a43,ANSSI-CC-2010/02,None,None,None -cd72fc2b62780a43,ANSSI-CC-2010/21,None,None,None -cdbbbad526f0aa13,BSI-DSZ-CC-1110-V4-2021,None,None,None -cdecc0eedcf05d2c,BSI-DSZ-CC-0285-2005,None,None,None -ce22219c2049985c,ANSSI-CC-2020/32,None,None,None -ce92195cf4a19084,BSI-DSZ-CC-0494-2008,None,None,None -cecb6a5d441fba98,BSI-DSZ-CC-0801-2012,None,None,None -cf1c544add4bf860,BSI-DSZ-CC-1137-2020,None,None,None -d063a93beba2dd65,BSI-DSZ-CC-0555-2009,None,None,None -d07b6e3036abb8f1,BSI-DSZ-CC-0645-2010,None,None,None -d07b6e3036abb8f1,BSI-DSZ-CC-0731-2011,None,None,None -d07b6e3036abb8f1,BSI-DSZ-CC-0741-2011,None,None,None -d07b6e3036abb8f1,BSI-DSZ-CC-0750-2011,None,None,None -d08b5ddedd34448d,ANSSI-CC-2018/35,None,None,None -d08b5ddedd34448d,BSI-DSZ-CC-0945-2017,None,None,None -d12a50d33b1c4253,CCEVS-VR-05-0094,None,None,None -d1fe94ee32cbbf05,BSI-DSZ-CC-0266-2005,None,None,None -d2bece2b0eebce5b,BSI-DSZ-CC-0879-V4-2020,None,None,None -d2bece2b0eebce5b,BSI-DSZ-CC-1071-V2-2019,None,None,None -d2d43918e5447c09,BSI-DSZ-CC-1156-V2-2021,None,None,None -d317c9c2e53ce427,BSI-DSZ-CC-0808-2013,None,None,None -d317c9c2e53ce427,BSI-DSZ-CC-0809-V2-2016,None,None,None -d317c9c2e53ce427,BSI-DSZ-CC-0978-2016,None,None,None -d32e5d4d5ff8b871,ANSSI-CC-2017/76,None,None,None -d32e5d4d5ff8b871,BSI-DSZ-CC-0891-V2-2016,None,None,None -d3323745a51a604d,BSI-DSZ-CC-0695-2011,None,None,None -d3323745a51a604d,BSI-DSZ-CC-0813-2012,None,None,None -d3323745a51a604d,BSI-DSZ-CC-0829-2012,None,None,None -d3420c2bf8974a5d,BSI-DSZ-CC-0782-V2-2015,None,None,None -d3420c2bf8974a5d,BSI-DSZ-CC-0967-2016,None,None,None -d34363168b1590bd,BSI-DSZ-CC-0964-V3-2017,None,None,None -d3987535a95fbffd,BSI-DSZ-CC-1040-2019,None,None,None -d3c2a59a2cffa30c,BSI-DSZ-CC-0555-2009,None,None,None -d3d257648d4c3105,ANSSI-CC-2010/02,None,None,None -d3d257648d4c3105,ANSSI-CC-2010/19,None,None,None -d3d257648d4c3105,ANSSI-CC-2010/56,None,None,None -d3d257648d4c3105,ANSSI-CC-2010/57,None,None,None -d4647f5217d84ba3,BSI-DSZ-CC-1040-2019,None,None,None -d4a1feebc1e1cb5d,BSI-DSZ-CC-0858-2013,None,None,None -d5187fc1cd590073,ANSSI-CC-2013/18,None,None,None -d67edade78d25033,ANSSI-CC-2009/27,None,None,None -d67edade78d25033,ANSSI-CC-2010/03,None,None,None -d69309b0a27c929c,BSI-DSZ-CC-1133-2020,None,None,None -d6ade0a63d0f9130,BSI-DSZ-CC-0897-V2-2014,None,None,None -d6ade0a63d0f9130,NSCIB-CC-12-36243-CR2,None,None,None -d6d129c94c368155,BSI-DSZ-CC-0217-2003,None,None,None -d71d0b082233758e,CRP260,None,None,None -d7501476836cf315,ANSSI-CC-2019/61,None,None,None -d75ff490c057793c,BSI-DSZ-CC-0727-2011,None,None,None -d80c26c2484dc87d,ANSSI-CC-2010/03,None,None,None -d9bfffa3cc6d1c53,BSI-DSZ-CC-0879-2014,None,None,None -da1139ecdc3512da,ANSSI-CC-2015/36,None,None,None -da2d3567a814d2d7,ANSSI-CC-2017/62,None,None,None -da2d3567a814d2d7,BSI-DSZ-CC-1059-V3-2019,None,None,None -da6845ee99cfa275,BSI-DSZ-CC-0266-2005,None,None,None -da6845ee99cfa275,BSI-DSZ-CC-0322-2005,None,None,None -da6845ee99cfa275,BSI-DSZ-CC-0399-2007,None,None,None -db28c1d7f201af1a,BSI-DSZ-CC-0640-2010,None,None,None -db28c203846775d8,ANSSI-CC-2014/06,None,None,None -db28c203846775d8,ANSSI-CC-2014/07,None,None,None -db28c203846775d8,BSI-DSZ-CC-0829-2012,None,None,None -db9267bdc717c801,BSI-DSZ-CC-0399-2007,None,None,None -db9267bdc717c801,BSI-DSZ-CC-0446-2007,None,None,None -dbbb56b48acbaf22,ANSSI-CC-2018/51,None,None,None -dbcf1311b3d95cdc,BSI-DSZ-CC-0332-2007,None,None,None -dbd62ef3bd952bad,BSI-DSZ-CC-0399-2007,None,None,None -dbe20d7c305b24eb,BSI-DSZ-CC-0750-V2-2014,None,None,None -dbe20d7c305b24eb,BSI-DSZ-CC-0858-2013,None,None,None -dbe20d7c305b24eb,NSCIB-CC-13-37762,None,None,None -dc51cf4e51f01086,NSCIB-CC-0351632-CR,None,None,None -dcd67613012044b3,BSI-DSZ-CC-0964-2015,None,None,None -dd4cfd03d9b3f25f,ANSSI-CC-2011/10,None,None,None -dd4cfd03d9b3f25f,BSI-DSZ-CC-0523-2008,None,None,None -dd4cfd03d9b3f25f,BSI-DSZ-CC-0626-2009,None,None,None -ddd9691cb063b07d,BSI-DSZ-CC-0973-V2-2016,None,None,None -ddd9691cb063b07d,BSI-DSZ-CC-0995-2018,None,None,None -ddd9691cb063b07d,BSI-DSZ-CC-0996-2018,None,None,None -de2ce0c1b72d5c90,BSI-DSZ-CC-0437-2008,None,None,None -de50abaf95e79d34,BSI-DSZ-CC-0813-2012,None,None,None -e0283686d26420d9,BSI-DSZ-CC-1021-2016,None,None,None -e0999b85d2f77733,BSI-DSZ-CC-0782-V2-2015,None,None,None -e0be32a08b66d264,BSI-DSZ-CC-0754-2012,None,None,None -e23f9c02819688f6,ANSSI-CC-2012/22,None,None,None -e247b50a301ce26c,BSI-DSZ-CC-1136-2021,None,None,None -e2a45ce49bb2b731,BSI-DSZ-CC-0829-2012,None,None,None -e2a45ce49bb2b731,BSI-DSZ-CC-0829-V2-2015,None,None,None -e3129aad84d5c40c,BSI-DSZ-CC-0837-V2-2014,None,None,None -e3231fac88242d81,BSI-DSZ-CC-0754-2012,None,None,None -e359ab2e436adf36,BSI-DSZ-CC-1040-2019,None,None,None -e359ab2e436adf36,BSI-DSZ-CC-1120-2020,None,None,None -e367ecaf05dacedb,BSI-DSZ-CC-0322-2005,None,None,None -e367ecaf05dacedb,BSI-DSZ-CC-0482-2008,None,None,None -e3a743ef04ecfb64,ANSSI-CC-2010/02,None,None,None -e4dc90de190437b8,CRP178,None,None,None -e4dc90de190437b8,CRP182,None,None,None -e501cafa3b025fb9,BSI-DSZ-CC-0750-2011,None,None,None -e501cafa3b025fb9,BSI-DSZ-CC-0858-2013,None,None,None -e501cafa3b025fb9,BSI-DSZ-CC-0866-2013,None,None,None -e501cafa3b025fb9,BSI-DSZ-CC-0866-V2-2014,None,None,None -e56d5d27cb2b3e6f,BSI-DSZ-CC-0403-2008,None,None,None -e56e14e8cbf89e3c,ANSSI-CC-2018/60,None,None,None -e56e14e8cbf89e3c,ANSSI-CC-2019/23,None,None,None -e5863682239df24b,BSI-DSZ-CC-0857-2013,None,None,None -e5b04a0a3872954f,ANSSI-CC-2011/07,None,None,None -e5b04a0a3872954f,ANSSI-CC-2012/48,None,None,None -e64266a9c13fc74e,ANSSI-CC-2014/81,None,None,None -e64266a9c13fc74e,BSI-DSZ-CC-0946-2014,None,None,None -e64266a9c13fc74e,BSI-DSZ-CC-0946-V2-2015,None,None,None -e64eedbb40d09a50,ANSSI-CC-2017/73,None,None,None -e6959c2b66202cb8,BSI-DSZ-CC-0891-V4-2019,None,None,None -e6aaffd588f2832e,ANSSI-CC-2013/56,None,None,None -e6aaffd588f2832e,ANSSI-CC-2014/83,None,None,None -e6dd3d435bd1b82e,BSI-DSZ-CC-0606-2010,None,None,None -e7a197d6aefb66df,BSI-DSZ-CC-0541-2010,None,None,None -e89a1a7817dae7e4,BSI-DSZ-CC-1089-2020,None,None,None -e8a37e48a8d34410,BSI-DSZ-CC-0466-2008,None,None,None -e8adf99d340489ce,ANSSI-CC-2017/76,None,None,None -e8adf99d340489ce,BSI-DSZ-CC-0891-V2-2016,None,None,None -e92ba97c2742e63d,ANSSI-CC-2019/33,None,None,None -e92ba97c2742e63d,ANSSI-CC-2020/26,None,None,None -e92ba97c2742e63d,BSI-DSZ-CC-1059-2018,None,None,None -e92ba97c2742e63d,BSI-DSZ-CC-1059-V3-2019,None,None,None -e93cb94a06c6957e,BSI-DSZ-CC-0964-V3-2017,None,None,None -e9845b4128afd538,BSI-DSZ-CC-1107-2020,None,None,None -e9c5d02aae54cdc5,BSI-DSZ-CC-0891-V4-2019,None,None,None -ea17788d9a9d12eb,CRP253,None,None,None -ea1b5ef156502d98,BSI-DSZ-CC-0862-2016,None,None,None -ea316d47c03f9fdf,BSI-DSZ-CC-0813-2012,None,None,None -ea330145d3ba738d,BSI-DSZ-CC-0322-2005,None,None,None -ea330145d3ba738d,BSI-DSZ-CC-0456-2009,None,None,None -ea4f0c0a7b82f526,BSI-DSZ-CC-0964-2015,None,None,None -eac2041efb4cf9f7,ANSSI-CC-2019/12,None,None,None -ec21d3c76c5ef65d,BSI-DSZ-CC-0904-V2-2021,None,None,None -ec21d3c76c5ef65d,BSI-DSZ-CC-1068-V2-2022,None,None,None -ec8a2a25cdbda634,ANSSI-CC-2020/07,None,None,None -ec8a2a25cdbda634,BSI-DSZ-CC-1110-V2-2019,None,None,None -ecee66b37d7576db,BSI-DSZ-CC-0244-2004,None,None,None -ed511caf8988dc45,BSI-DSZ-CC-0827-2013,None,None,None -ee35d872179a1397,BSI-DSZ-CC-0341-2006,None,None,None -ee35d872179a1397,BSI-DSZ-CC-0379-2006,None,None,None -ee3a996c18ace68e,BSI-DSZ-CC-0897-V2-2014,None,None,None -ee4e169753085913,BSI-DSZ-CC-0645-2010,None,None,None -ee4e169753085913,BSI-DSZ-CC-0682-2010,None,None,None -ee6718f7a7e3d30b,BSI-DSZ-CC-0266-2005,None,None,None -ee6718f7a7e3d30b,BSI-DSZ-CC-0338-2005,None,None,None -ef02bc67ebead2a2,BSI-DSZ-CC-0645-2010,None,None,None -ef02bc67ebead2a2,BSI-DSZ-CC-0675-2011,None,None,None -ef02bc67ebead2a2,BSI-DSZ-CC-0750-2011,None,None,None -f0bd6a29eee94a2c,ANSSI-CC-2018/08,None,None,None -f1300cb717ea9335,BSI-DSZ-CC-0900-2014,None,None,None -f14484e630df1dec,ANSSI-CC-2011/07,None,None,None -f14484e630df1dec,ANSSI-CC-2011/77,None,None,None -f14484e630df1dec,ANSSI-CC-2012/48,None,None,None -f14484e630df1dec,ANSSI-CC-2013/18,None,None,None -f1a73b2232787b26,ANSSI-CC-2010/02,None,None,None -f1a73b2232787b26,BSI-DSZ-CC-0664-2011,None,None,None -f1b9158cc2b98388,BSI-DSZ-CC-0782-2012,None,None,None -f1b9158cc2b98388,CRP279,None,None,None -f229a8078cb4ff82,BSI-DSZ-CC-1059-V2-2019,None,None,None -f2327302df1aa660,ANSSI-CC-2012/70,None,None,None -f2516946c797b1a0,ANSSI-CC-2013/42,None,None,None -f2516946c797b1a0,BSI-DSZ-CC-0829-2012,None,None,None -f2e7c6dfd6a431c8,BSI-DSZ-CC-0782-V5-2020,None,None,None -f2eb0ef953f3f147,ANSSI-CC-2015/45,None,None,None -f2eb0ef953f3f147,BSI-DSZ-CC-0782-V2-2015,None,None,None -f2fd56c195593dd8,ANSSI-CC-2020/94,None,None,None -f34df0dd7011366e,ANSSI-CC-2016/58,None,None,None -f34df0dd7011366e,ANSSI-CC-2017/61,None,None,None -f3bedaa5ffeeadb1,ANSSI-CC-2015/36,None,None,None -f4b4b026e87b5086,ANSSI-CC-2019/01,None,None,None -f4da9e13977580f4,BSI-DSZ-CC-0916-2015,None,None,None -f569fb2a6ddae452,BSI-DSZ-CC-0410-2007,None,None,None -f569fb2a6ddae452,BSI-DSZ-CC-0446-2007,None,None,None -f588fbe6fd36bc6b,BSI-DSZ-CC-0312-2005,None,None,None -f588fbe6fd36bc6b,BSI-DSZ-CC-0338-2005,None,None,None -f5a53fa6d2c3d9b2,ANSSI-CC-2017/11,None,None,None -f621ca835cf98a6d,BSI-DSZ-CC-0622-2012,None,None,None -f67f3737a3b47208,BSI-DSZ-CC-0782-2012,None,None,None -f68006f6c05e1211,BSI-DSZ-CC-0410-2007,None,None,None -f68006f6c05e1211,BSI-DSZ-CC-0411-2007,None,None,None -f731aaaab07bdca2,BSI-DSZ-CC-0973-2016,None,None,None -f731aaaab07bdca2,BSI-DSZ-CC-0973-V2-2016,None,None,None -f7941427d5fa40ea,BSI-DSZ-CC-0374-2006,None,None,None -f7cbb33cc639b5b5,BSI-DSZ-CC-0438-2007,None,None,None -f7cbb33cc639b5b5,BSI-DSZ-CC-0501-2008,None,None,None -f8a3fde3a557f44d,ANSSI-CC-2015/73,None,None,None -f8a3fde3a557f44d,BSI-DSZ-CC-0917-2014,None,None,None -f96bfb09bb1bd640,ANSSI-CC-2019/41,None,None,None -f96bfb09bb1bd640,ANSSI-CC-2020/07,None,None,None -f96bfb09bb1bd640,BSI-DSZ-CC-1110-V4-2021,None,None,None -f9d96d0eb2bc85cc,BSI-DSZ-CC-0750-V2-2014,None,None,None -f9d96d0eb2bc85cc,BSI-DSZ-CC-0858-2013,None,None,None -f9d96d0eb2bc85cc,BSI-DSZ-CC-0912-2014,None,None,None -f9d96d0eb2bc85cc,BSI-DSZ-CC-0913-2014,None,None,None -f9d96d0eb2bc85cc,BSI-DSZ-CC-0914-2014,None,None,None -f9d96d0eb2bc85cc,NSCIB-CC-13-37760-CR2,None,None,None -f9d9fe0dd9f7f110,BSI-DSZ-CC-0523-2008,None,None,None -f9e9366287e79e4d,BSI-DSZ-CC-0470-2008,None,None,None -f9fb8170cecedc23,BSI-DSZ-CC-1107-V3-2022,None,None,None -f9fb8170cecedc23,NSCIB-CC-0289060-CR,None,None,None -fa8384342c25ec59,BSI-DSZ-CC-0955-V2-2016,None,None,None -fb1fff87c87b82d6,ANSSI-CC-2012/70,None,None,None -fb1fff87c87b82d6,BSI-DSZ-CC-0894-2014,None,None,None -fb2cfdfef4f93536,BSI-DSZ-CC-0951-V2-2017,None,None,None -fb8010928a4bb56b,BSI-DSZ-CC-0837-V2-2014,None,None,None -fb8010928a4bb56b,BSI-DSZ-CC-0892-2014,None,None,None -fb8010928a4bb56b,BSI-DSZ-CC-0893-2014,None,None,None -fb8010928a4bb56b,NSCIB-CC-12-36243-CR2,None,None,None -fbdaf22c4f50391c,BSI-DSZ-CC-0891-V3-2018,None,None,None -fbe41655a05a01ff,BSI-DSZ-CC-0718-2012,None,None,None -fca98ecd003e1b82,ANSSI-CC-2009/11,None,None,None -fca98ecd003e1b82,BSI-DSZ-CC-0421-2008,None,None,None -fce7e01f2cd0cac0,ANSSI-CC-2011/10,None,None,None -fce7e01f2cd0cac0,ANSSI-CC-2011/12,None,None,None -fce7e01f2cd0cac0,BSI-DSZ-CC-0523-2008,None,None,None -fce7e01f2cd0cac0,BSI-DSZ-CC-0630-2010,None,None,None -fd3f988b7a505c45,BSI-DSZ-CC-0169-2002,None,None,None -fd583001a87023fd,BSI-DSZ-CC-0481-2008,None,None,None -fd583001a87023fd,BSI-DSZ-CC-0579-2009,None,None,None -fd583001a87023fd,CCEVS-VR-07-0054,None,None,None -fd966f807d3ba9fe,BSI-DSZ-CC-0813-2012,None,None,None -fd966f807d3ba9fe,BSI-DSZ-CC-0850-2013,None,None,None -ff0ffb9a32873433,BSI-DSZ-CC-0266-2005,None,None,None -ff0ffb9a32873433,BSI-DSZ-CC-0322-2005,None,None,None -ff45605151ec4eb1,ANSSI-CC-2013/39,None,None,None -ff45605151ec4eb1,BSI-DSZ-CC-0606-2010,None,None,None -ff45605151ec4eb1,BSI-DSZ-CC-0640-2010,None,None,None -ff45605151ec4eb1,BSI-DSZ-CC-0813-2012,None,None,None -ffc97a8fff838fa8,BSI-DSZ-CC-0266-2005,None,None,None -ffc97a8fff838fa8,BSI-DSZ-CC-0322-2005,None,None,None -ffc97a8fff838fa8,BSI-DSZ-CC-0538-2008,None,None,None -ffd213e55eee8cc8,BSI-DSZ-CC-0976-V2-2018,None,None,None -ffd213e55eee8cc8,BSI-DSZ-CC-1110-V2-2019,None,None,None -ffe4328dcdc6b538,BSI-DSZ-CC-0715-2011,None,None,None -fff289a66cb9ce4b,NSCIB-CC-0351632-CR,None,None,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv b/src/sec_certs/data/reference_annotations/manual_annotations/train.csv deleted file mode 100644 index 2cab70dd..00000000 --- a/src/sec_certs/data/reference_annotations/manual_annotations/train.csv +++ /dev/null @@ -1,197 +0,0 @@ -dgst,canonical_reference_keyword,source,label,comment -00a2b91e5b58cef1,ANSSI-CC-2014/23,None,None,None -00ad121fa63e5dfb,BSI-DSZ-CC-0973-V2-2016,None,None,None -00ad121fa63e5dfb,BSI-DSZ-CC-0996-2018,None,None,None -00b539d629e60965,CCEVS-VR-02-0025,None,None,None -00eb3e5c7a4537b1,BSI-DSZ-CC-0322-2005,None,None,None -00eb3e5c7a4537b1,BSI-DSZ-CC-0399-2007,None,None,None -00efeb17bcaafce6,ANSSI-CC-2009/34,report,component_used,None -017f3cea8275a3e1,ANSSI-CC-2017/23,None,None,None -01ce6f422bec823d,BSI-DSZ-CC-1040-2019,None,None,None -01dc77d7f8f94d97,ANSSI-CC-2016/78,None,None,None -01dc77d7f8f94d97,ANSSI-CC-2017/39,None,None,None -01e5c6d31f763f56,BSI-DSZ-CC-0917-2014,None,None,None -01e8805514b4ef67,BSI-DSZ-CC-0891-V3-2018,report,component_used,None -0208e6b60a0b1a32,ANSSI-CC-2016/01,None,None,None -02c6fbcd90dcf05a,BSI-DSZ-CC-1107-V3-2022,None,None,None -02cb3c385865ad2c,BSI-DSZ-CC-1184-2022,None,None,None -0319b03323f257d1,BSI-DSZ-CC-1118-2020,None,None,None -0319b03323f257d1,BSI-DSZ-CC-1121-2019,None,None,None -032faef0207b715f,BSI-DSZ-CC-0837-V2-2014,None,None,None -037577fc2019fcfa,ANSSI-CC-2021/49,report,previous_version,None -04c5a5a66fb7f89a,BSI-DSZ-CC-0827-V2-2014,None,None,None -04cb360502da6492,BSI-DSZ-CC-0344-2005,None,None,None -054bd8196e015197,ANSSI-CC-2011/07,None,None,None -055985699aee1e09,ANSSI-CC-2010/02,report,component_used,None -05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,report,component_used,None -05d96aada7bb839c,BSI-DSZ-CC-0223-2003,None,None,None -06b24bc51eb69188,BSI-DSZ-CC-0813-2012,None,None,None -06e505abb8dad1b8,ANSSI-CC-2017/54,report,component_used,None -06e70fa70393d60f,ANSSI-CC-2017/07,None,None,None -06e70fa70393d60f,BSI-DSZ-CC-0891-V2-2016,None,None,None -075ddd00e07fa7ed,ANSSI-CC-2020/76,None,None,None -075ddd00e07fa7ed,BSI-DSZ-CC-1136-2021,None,None,None -07ce7e4844f62c5c,ANSSI-CC-2017/73,None,None,None -07ce7e4844f62c5c,BSI-DSZ-CC-1070-2018,None,None,None -080aa6966d757f05,ANSSI-CC-2010/02,None,None,None -080aa6966d757f05,ANSSI-CC-2010/05,None,None,None -08408cbc394e7421,ANSSI-CC-2017/59,None,None,None -08408cbc394e7421,BSI-DSZ-CC-0782-V4-2018,None,None,None -09220515a7edf3f9,BSI-DSZ-CC-1110-V2-2019,None,None,None -096b05114ee5e09f,ANSSI-CC-2008/45,None,None,None -096b05114ee5e09f,BSI-DSZ-CC-0410-2007,None,None,None -096b05114ee5e09f,BSI-DSZ-CC-0411-2007,None,None,None -096ccb6bcb3d2731,BSI-DSZ-CC-0564-2009,None,None,None -09b07fbbb24e1b04,BSI-DSZ-CC-0964-2015,None,None,None -09f539aedefa9756,ANSSI-CC-2008/05,None,None,None -09f539aedefa9756,ANSSI-CC-2009/11,None,None,None -09f539aedefa9756,ANSSI-CC-2009/41,None,None,None -0a6e50445a906052,BSI-DSZ-CC-0891-V2-2016,None,None,None -0a6e50445a906052,BSI-DSZ-CC-0945-2017,None,None,None -0aad0176e35b3749,BSI-DSZ-CC-0477-2007,None,None,None -0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None -0bd651fcbb86f2a9,ANSSI-CC-2020/74,None,None,None -0bd651fcbb86f2a9,BSI-DSZ-CC-1136-2021,None,None,None -0bf7a19b22163465,ANSSI-CC-2015/80,report,evaluation_reused,None -0bf7a19b22163465,ANSSI-CC-2016/44,report,evaluation_reused,unclear whether component used -0c9e215997d96f44,BSI-DSZ-CC-0555-2009,None,None,None -0cd406e7c5013b74,BSI-DSZ-CC-0304-2006,None,None,None -0d4a95a8b52b787b,BSI-DSZ-CC-0829-2012,None,None,None -0da9d81c9f563df5,BSI-DSZ-CC-0382-2007,None,None,None -0ee132dec087b772,ANSSI-CC-2011/07,None,None,None -0ef56cba2362e463,ANSSI-CC-2015/42,None,None,None -0f1b31ac93459d1f,BSI-DSZ-CC-0720-2011,None,None,None -0f3900cdcd0c7f3e,NSCIB-CC-66030-CR5,report,component_used,None -0f684159ad31f883,BSI-DSZ-CC-0555-2009,None,None,None -0fe0bb1054f49b7f,CCEVS-VR-04-0092,None,None,None -0ff82b1cc24b3105,BSI-DSZ-CC-0302-2005,None,None,None -10233188161519ee,ANSSI-CC-2019/48,None,None,None -10233188161519ee,ANSSI-CC-2020/26,None,None,None -10233188161519ee,BSI-DSZ-CC-1059-2018,None,None,None -10233188161519ee,BSI-DSZ-CC-1059-V3-2019,None,None,None -102707a8c4d8a1b0,ANSSI-CC-2019/35,None,None,None -102707a8c4d8a1b0,ANSSI-CC-2020/26,None,None,None -102707a8c4d8a1b0,BSI-DSZ-CC-1059-2018,None,None,None -102707a8c4d8a1b0,BSI-DSZ-CC-1059-V3-2019,None,None,None -102bbdfcd696d7a9,ANSSI-CC-2018/13,None,None,None -102bbdfcd696d7a9,ANSSI-CC-2019/01,None,None,None -107c348f4ba0a41c,BSI-DSZ-CC-1135-2020,None,None,None -109eb2158ca6a2f9,BSI-DSZ-CC-0410-2007,None,None,None -10af4a6378d0fb7a,ANSSI-CC-2014/07,None,None,None -10af4a6378d0fb7a,ANSSI-CC-2014/86,None,None,None -10af4a6378d0fb7a,BSI-DSZ-CC-0837-V2-2014,None,None,None -10b17081dd7cad8f,ANSSI-CC-2015/36,None,None,None -10b17081dd7cad8f,ANSSI-CC-2017/39,None,None,None -112fe9f123e277ee,ANSSI-CC-2010/02,None,None,None -112fe9f123e277ee,ANSSI-CC-2010/19,None,None,None -114e39b4da949e6d,ANSSI-CC-2020/08,None,None,None -114e39b4da949e6d,BSI-DSZ-CC-1110-V2-2019,None,None,None -1172a88e009dc248,CRP257,None,None,None -1172a88e009dc248,CRP281,None,None,None -1188aebd8681e4fe,BSI-DSZ-CC-0465-2008,None,None,None -122245dd15683e26,ANSSI-CC-2007/08,None,None,None -122245dd15683e26,BSI-DSZ-CC-0269-2006,None,None,None -123c94418ddf1ad8,BSI-DSZ-CC-0367-2006,None,None,None -12e20466a0f08342,ANSSI-CC-2016/79,report,evaluation_reused,unclear whether component used -131ec924c42ef90b,ANSSI-CC-2017/61,None,None,None -135ca3602f0005bf,NSCIB-CC-16-99111-CR2,None,None,None -13d95239226aa537,BSI-DSZ-CC-0835-V2-2017,report,component_shared,"BAC, EAC thingy" -13e5b88ad3becb77,BSI-DSZ-CC-0322-2005,None,None,None -13e5b88ad3becb77,BSI-DSZ-CC-0344-2005,None,None,None -1468948b4f0509ab,BSI-DSZ-CC-0813-2012,None,None,None -157b741a0f529a0a,ANSSI-CC-2009/37,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0555-2009,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0633-2010,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0675-2011,None,None,None -157f811757a0b29e,BSI-DSZ-CC-0745-2012,None,None,None -158950ed4bc35274,ANSSI-CC-2015/36,None,None,None -158950ed4bc35274,ANSSI-CC-2015/80,None,None,None -158950ed4bc35274,ANSSI-CC-2016/33,None,None,None -16bdbde359584f99,ANSSI-CC-2017/61,report,component_used,None -17a23970b35a4f44,BSI-DSZ-CC-0782-V5-2020,report,component_used,None -19c60923e31777e7,ANSSI-CC-2017/49,report,component_used,None -1b8705a0486b3be1,NSCIB-CC-0209053-CR,report,previous_version,None -1e40bd733acd56bc,BSI-DSZ-CC-0782-V4-2018,target,component_used,None -1f9f4b843070fc1b,BSI-DSZ-CC-1105-2020,report,component_used,None -27d11629261d8806,ANSSI-CC-2014/59,report,component_used,None -28e70f46ba4394dd,ANSSI-CC-2012/08,report,component_used,None -2c15dfc106ebf8d8,ANSSI-CC-2012/24,report,component_used,None -30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,report,component_used,None -30f71b100c5cebae,ANSSI-CC-2015/15,report,component_used,None -317713948b4473b3,BSI-DSZ-CC-0798-2012,report,component_shared,None -34370b67b5e675c3,ANSSI-CC-2013/47,report,component_used,None -3515801dee00995f,BSI-DSZ-CC-0447-2008,report,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 -36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,report,component_used,None -371dca18821f7714,BSI-DSZ-CC-0945-2017,report,component_used,None -3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,report,recertification,None -42799014c183e1b3,BSI-DSZ-CC-0312-2005,report,component_used,None -42799014c183e1b3,BSI-DSZ-CC-0429-2007,report,recertification,None -463ecd64b7506048,ANSSI-CC-2018/52,report,component_used,None -4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,report,component_used,None -4bede5d2af4cb11f,ANSSI-CC-2019/59,report,evaluation_reused,None -4f5f41ecf7517e63,ANSSI-CC-2016/80,report,evaluation_reused,None -54273dd266fce692,ANSSI-CC-2015/15,report,component_used,None -5a66027d34aafc52,BSI-DSZ-CC-1019-V2-2019,target,component_used,None -5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,target,component_used,unclear -61832cb4291c343f,BSI-DSZ-CC-0955-2016,report,component_used,None -627bfd69c2a831cf,ANSSI-CC-2014/37,report,evaluation_reused,None -680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,report,component_used,None -691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,report,component_used,None -6a999675c9422dfb,BSI-DSZ-CC-0353-2006,report,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 -6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,target,component_used,None -6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,report,component_used,None -744a7a202d909323,ANSSI-CC-2017/24,report,component_used,None -7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,report,recertification,None -845bb039719ac5d8,ANSSI-CC-2010/40,report,previous_version,None -84a75b6fc33b669e,ANSSI-CC-2014/08,report,evaluation_reused,None -8ba22f6c9651edc3,ANSSI-CC-2013/55,report,component_used,None -8bdb610131555c12,BSI-DSZ-CC-0645-2010,report,component_used,None -921e042759b30033,BSI-DSZ-CC-0978-2016,report,component_used,composition scheme “[COMP]” mentioned -9529eb793550093c,ANSSI-CC-2018/40,report,previous_version,None -97d00a9f198a6e57,ANSSI-CC-2015/66,report,component_used,None -a07548a64fac5794,BSI-DSZ-CC-0350-2007,report,recertification,None -a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,report,component_used,None -a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,report,previous_version,None -a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,report,component_used,None -a867d281d34d34b7,BSI-DSZ-CC-0976-2015,report,recertification,None -a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,report,recertification,None -aa096ffde94b953b,BSI-DSZ-CC-0411-2007,report,component_used,None -ab3af998dff7a2ef,ANSSI-CC-2017/47,report,component_used,None -ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,report,component_used,None -aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,report,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 -af61a31e3fd0d6f0,ANSSI-CC-2010/02,report,evaluation_reused,None -b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,None,unclear -b42009eb34746731,ANSSI-CC-2021/50,report,previous_version,maybe previous_version -b61363e51ed90c7a,ANSSI-CC-2019/12,report,component_used,None -bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,report,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" -c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,report,component_used,None -c175537338906951,ANSSI-CC-2016/70,report,component_shared,None -c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,report,evaluation_reused,None -c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,None,"unclear, mentions just ToE delivery" -c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,report,recertification,None -c96343baf608174d,BSI-DSZ-CC-0645-2010,report,component_used,None -c96343baf608174d,BSI-DSZ-CC-0750-2011,report,component_used,None -cacdfdafce47678c,ANSSI-CC-2014/59,target,component_used,None -cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,report,component_used,None -d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,report,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert -d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,report,recertification,None -d49988efd778ca9d,ANSSI-CC-2010/02,report,component_used,None -d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,report,recertification,None -dba4b5a166f9456a,ANSSI-CC-2014/76,report,recertification,None -df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,report,component_used,None -e0265f0fb8e196c0,BSI-DSZ-CC-0827-V8-2020,None,None,unclear -e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,report,recertification,None -e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,report,component_used,the previously certified M7892 G12 component -e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,report,component_used,None -ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,report,recertification,None -ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,target,component_used,None -ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,report,component_used,None -ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,report,component_used,None -ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,report,recertification,None -ee1c6dd97918d74a,ANSSI-CC-2012/71,target,evaluation_reused,None -f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,report,component_used,None -f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,report,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 -f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,target,component_used,None -f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,report,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 -f869956d14ea1694,ANSSI-CC-2012/68,report,component_used,None -fb9ec9e846ea4e87,ANSSI-CC-2017/47,report,component_used,None diff --git a/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv b/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv deleted file mode 100644 index e1896fe3..00000000 --- a/src/sec_certs/data/reference_annotations/manual_annotations/valid.csv +++ /dev/null @@ -1,108 +0,0 @@ -dgst,canonical_reference_keyword,source,label,comment -a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,report,component_used,None -657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,report,recertification,None -0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,report,component_used,None -983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,report,recertification,None -8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,report,recertification,None -774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,report,evaluation_reused,None -c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,None,None,None -a5adb726852f5cc5,ANSSI-CC-2009/37,report,component_used,None -4d983357fd464bb5,BSI-DSZ-CC-1107-V2-2021,report,component_used,None -cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,report,component_used,None -24de96ea505af909,ANSSI-CC-2021/29,report,component_used,None -2793414918738c7f,BSI-DSZ-CC-0857-2013,target,component_used,None -b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,report,recertification,None -d96438a7165e5d82,BSI-DSZ-CC-0410-2007,report,component_used,None -4b3f577896c80f8a,ANSSI-CC-2015/80,report,evaluation_reused,None -22388445fe620ac0,ANSSI-CC-2014/20,report,evaluation_reused,None -2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,report,recertification,None -6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,report,component_used,None -120a49c0aa284f68,BSI-DSZ-CC-0813-2012,report,component_used,None -994ac7d986a99e50,BSI-DSZ-CC-0640-2010,report,component_used,None -16abf8ee0697e64f,ANSSI-CC-2014/61,report,component_shared,None -f05de4db911ec8af,CRP243,report,previous_version,None -38f79f1a48e7844f,ANSSI-CC-2017/47,report,component_used,None -fe55e99f4f3dc195,ANSSI-CC-2013/33,report,evaluation_reused,None -fc875e0381a28435,BSI-DSZ-CC-0432-2007,report,recertification,None -c23bb542201aa511,BSI-DSZ-CC-0829-2012,report,component_used,None -2532fe8c7214c485,BSI-DSZ-CC-0675-2011,report,component_shared,None -2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,report,component_used,None -833ff87d69cff5db,383-4-184,None,None,unclear -c7c1215819127a1c,BSI-DSZ-CC-0322-2005,report,component_used,None -647d17d44745a532,BSI-DSZ-CC-0904-2015,report,recertification,None -4489bfc781a82281,BSI-DSZ-CC-0817-2013,report,recertification,None -94fb86d31446e531,BSI-DSZ-CC-0958-2015,report,component_used,None -37bae9eaf342542b,BSI-DSZ-CC-0349-2006,report,evaluation_reused,None -5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,report,component_used,None -5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,report,recertification,None -d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,report,component_used,None -0f53257eb74bee54,BSI-DSZ-CC-0417-2008,report,component_used,unclear -84d6a370e4f38ddc,ANSSI-CC-2019/12,report,component_used,None -1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,report,recertification,None -d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,report,component_used,None -38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,report,recertification,None -e6057ffc5085a192,ANSSI-CC-2014/48,report,evaluation_reused,None -db7627e56b8621b1,BSI-DSZ-CC-0639-2010,report,component_used,None -4b3f577896c80f8a,ANSSI-CC-2016/44,report,evaluation_reused,None -5f3c9c6bf76ba72d,ANSSI-CC-2018/51,report,component_used,None -22d6578b30f5a227,BSI-DSZ-CC-1107-2020,report,component_used,None -a0c38b4389ad7cf7,ANSSI-CC-2018/12,report,component_used,None -19c7a1b1a2df87d7,BSI-DSZ-CC-0945-V2-2018,report,component_used,None -f654c82bca57cf62,BSI-DSZ-CC-0957-V2-2016,None,None,unclear -2f0be9733140e2c6,BSI-DSZ-CC-0977-V2-2019,report,component_used,None -3357c28c91cc2ed1,ANSSI-CC-2016/44,report,evaluation_reused,None -f6317f134f532696,ANSSI-CC-2017/24,report,previous_version,None -234da54efbcf734b,BSI-DSZ-CC-0917-2014,report,component_used,None -1b20b1629ba7d3a6,ANSSI-CC-2012/72,target,component_used,None -2532fe8c7214c485,BSI-DSZ-CC-0633-2010,report,component_used,None -b1ef468658a9a0c1,NSCIB-CC-0448219-CR,report,evaluation_reused,None -35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,report,recertification,None -edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,report,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. -Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of -technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the -TOE was re-evaluated because of changes on top layers for modified module -packaging requirements. Additional package types (MOB4 and complete -passport inlay) and the relevant production sites were included in the evaluation -process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. The additional production steps do not influence the configuration of the -TOE itself. The Security Target was updated. " -3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,report,component_used,None -0fe8df0e85116b61,ANSSI-CC-2010/33,report,previous_version,None -bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,report,recertification,None -b00bb452e7a8176c,ANSSI-CC-2016/57,report,previous_version,None -7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,report,recertification,None -81273108dd167b98,BSI-DSZ-CC-0523-2008,report,recertification,None -63c1045055ec3b58,BSI-DSZ-CC-0266-2005,report,component_used,None -a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,report,component_used,None -4c017087976d05eb,ANSSI-CC-2020/45,report,evaluation_reused,None -f4d510a39278e687,ANSSI-CC-2018/32,report,component_used,None -c23bb542201aa511,ANSSI-CC-2014/14,report,previous_version,None -625b243397001f01,BSI-DSZ-CC-1059-2018,target,component_used,None -cae968d4926648bd,BSI-DSZ-CC-0754-2012,report,recertification,None -000176284683faa2,BSI-DSZ-CC-0640-2010,report,component_used,None -66faa094a9333d1a,ANSSI-CC-2014/86,report,evaluation_reused,None -a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,report,component_used,None -031667f4e242da61,CCEVS-VR-07-0054,report,component_used,None -459539da9315ccac,ANSSI-CC-2021/30,report,previous_version,None -69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,report,component_used,None -a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,report,component_used,None -50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,report,component_used,None -5cf51622fafaf868,BSI-DSZ-CC-1005-2016,report,recertification,None -1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,report,component_used,None -0c7ef6c32cbdee47,ANSSI-CC-2017/61,report,component_used,None -8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,report,component_used,None -6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,report,component_used,None -4962c0d8106b8b5e,ANSSI-CC-2015/37,report,evaluation_reused,None -e6057ffc5085a192,BSI-DSZ-CC-0829-2012,report,component_used,None -03fad65e5af65088,BSI-DSZ-CC-0696-2011,report,recertification,None -720aab4f782f4c60,BSI-DSZ-CC-0555-2009,report,component_used,None -9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,report,recertification,None -d55eb8c97d71e843,ANSSI-CC-2021/49,report,previous_version,None -73e0b884cd04d9bc,NSCIB-CC-12-36243,report,component_used,None -2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,report,component_used,None -f6a89befd13a643f,BSI-DSZ-CC-0951-V4-2019,report,component_used,None -f00a01940be6481d,BSI-DSZ-CC-0837-V2-2014,report,component_used,None -2f43fe24bd8e91e1,ANSSI-CC-2020/65,report,component_used,None -d96438a7165e5d82,BSI-DSZ-CC-0411-2007,report,component_used,None -7623950728f52a71,ANSSI-CC-2016/06,report,component_used,None -ba69597434946260,BSI-DSZ-CC-0293-2005,report,component_used,None -50aef770cb4f028f,BSI-DSZ-CC-1059-2018,report,component_used,None diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md index b3a240a9..8ed7eecd 100644 --- a/src/sec_certs/data/reference_annotations/readme.md +++ b/src/sec_certs/data/reference_annotations/readme.md @@ -4,6 +4,10 @@ This folder contains data and the methodology (presented below) related to learn - The folder [split](split) contains split of the CC Dataset to `train/valid/test` splits for learning. - The csv file [manually_annotated_references.csv](./manually_annotated_references.csv) contains manually acquired labels to references obtained with the methodology outlined below. +- The folder `adam` contains manual annotations created by Adam +- The folder `jano` contains manual annotations created by Jano +- The folder `conflicts` contains conflicting annotations between Adam and Jano, as weel as their resolution +- The contents of the `final` folder can thus be obtained by taking annotations either from `adam` or `jano` folder and masking them by `resolution_label` from `conflicts` folder. ### Reference taxonomy @@ -12,31 +16,27 @@ After manually inspecting random certificates, we have identified the following - **Component used**: The referenced certificate is a component used in the examined certificate (e.g., IC used by a smartcard). Some evaluation results were likely shared/re-used. - **Component shared**: The referenced certificate shares some components with the examined certificate. Some evaluation results were likely shared/re-used. - **Evaluation reused**: The evaluation results of the referenced certificate were used for evaluation of the examined certificate, due to reasons that could not be resolved. -- **Re-certification**: The examined certificate is a re-certification of the referenced certificate. +- **Recertification**: The examined certificate is a re-certification of the referenced certificate. - **Previous version**: The product in the referenced certificate is a previous version of the product in the examined certificate and the re-certification is not explicitly mentioned. -- **Unknown**: The annotator could not assign any of the previous contexts. +- **None**: The annotator could not assign any of the previous contexts. +- **Irrelevant**: The reference is irrelevant to the studied certificate (typo, left-out reference from a template, ...) These can be further merged into the following super-categories: -- **Component used or shared** -- **Previous version of re-certification** -- **Evaluation reused** - these cases should be manually visited -- **Unknown** +- **Some sub-component relationship** `component_used`, `component_shared`, and `evaluation_reused` +- **Previous version**: `previous_version` and `recertification` +- **None**: `None` or `irrelevant` ### Reference classification methodology **Data splits and manual annotations**: -1. Inspect random certificates (>100) and capture the observed relations into reference taxonomy -2. Split all certificates for which we register a direct outgoing reference in either security target or certification report into `train/valid/test` splits in `30/20/50` fashion. - - See [split](split/) -3. Label all references in as follows: - - Extract the text segments (using `ReferenceSegmentExtractor`) related to the references - - Use label-studio `Natural Language Processing -> text classification` setup to assign 1 label to each of the references. - - All text segments both from certification report and security target are displayed for the given instance - - The instances labeled with `Unknown` are re-visited and labeled after manual inspection of both certification report and security target pdfs - - Artifacts of the referenced certificate are not examined. - - The labeling is done by a pair of co-authors, the inter-annotator agreement is measured with Cohen's Kappa +1. Two authors inspect random certificates (~100) and capture the observed relations into reference taxonomy +2. Split all certificates for which we register a direct outgoing reference in either security target or certification report into `train/valid/test` splits in `30/20/50` fashion (see [split](split/)). +3. Sample 100 train, 100 valid, 200 test pairs of `(dgst, canonical_reference_keyword)` for manual annotations. +4. Two co-authors independently assign each of these pairs with a single label from the reference taxonomy. +5. Measure the inter-annotator agreement with Cohen's Kappa. +6. Resolve conflicts in the annotations in a meeting held by the co-authors. Use this consensual annotations for training and evaluation described below. **Learning the annotations**: diff --git a/src/sec_certs/model/references/segment_extractor.py b/src/sec_certs/model/references/segment_extractor.py index 4839a0b4..bf6096bc 100644 --- a/src/sec_certs/model/references/segment_extractor.py +++ b/src/sec_certs/model/references/segment_extractor.py @@ -290,7 +290,7 @@ class ReferenceSegmentExtractor: .assign(label=lambda df_: df_.label.str.replace(" ", "_").str.upper(), split=split_name) ) - annotations_directory = Path(str(files("sec_certs.data") / "reference_annotations/manual_annotations/")) + annotations_directory = Path(str(files("sec_certs.data") / "reference_annotations/final/")) df_annot = pd.concat( [ load_single_df(annotations_directory / "train.csv", "train"), -- cgit v1.3.1 From 4ae9bec3f7ce7c8a4666d16dc23601a4ef000aba Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 Oct 2023 10:03:05 +0200 Subject: add inter-annotator agreement --- pyproject.toml | 2 +- src/sec_certs/data/reference_annotations/readme.md | 39 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/pyproject.toml b/pyproject.toml index 938cf8aa..8a271bcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ 'sec_certs.config' = ["settings.yaml", "settings-schema.json"] 'sec_certs.data' = [ "reference_annotations/split/*.json", - "reference_annotations/manual_annotations/*.csv", + "reference_annotations/manual_annotations/final/*.csv", ] diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md index 8ed7eecd..b0d978e6 100644 --- a/src/sec_certs/data/reference_annotations/readme.md +++ b/src/sec_certs/data/reference_annotations/readme.md @@ -48,3 +48,42 @@ These can be further merged into the following super-categories: 4. Train a sentence transformer with the same soft-voting layer on top of that. 5. Finetune hyperparameters. 6. Evaluate on test set. + + +## Inter-annotator agreement + +The inter-annotator agreement is measured both with Cohen's Kappa and with percentage. The results are as follows: + +| Cohen's Kappa | Percentage | +|---------------|------------| +| 0.71 | 0.82 | + +The code used to measure the agreement is: + +```python +import pandas as pd +from pathlib import Path +from sklearn.metrics import cohen_kappa_score + +def load_all_dataframes(base_folder: Path) -> pd.DataFrame: + splits = ["train", "valid", "test"] + + df_train, df_valid, df_test = pd.DataFrame(), pd.DataFrame(), pd.DataFrame() + for split in splits: + df = pd.read_csv(base_folder / f"{split}.csv") + if split == "train": + df_train = df + elif split == "valid": + df_valid = df + else: + df_test = df + + return pd.concat([df_train, df_valid, df_test]) + +adam_df = load_all_dataframes(Path("./src/sec_certs/data/reference_annotations/adam")) +jano_df = load_all_dataframes(Path("./src/sec_certs/data/reference_annotations/jano")) +agreement_series = adam_df.label == jano_df.label + +print(f"Cohen's Kappa: {cohen_kappa_score(adam_df.label, jano_df.label)}") +print(f"Percentage agreement: {agreement_series.loc[agreement_series == True].count() / agreement_series.count()}") +``` -- cgit v1.3.1 From 36d48ebf09e41d02502c18682ee23e60ef9b2eec Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 Oct 2023 10:14:51 +0200 Subject: improve write-up of reference annotation methodology --- src/sec_certs/data/reference_annotations/readme.md | 61 +++++++++++----------- 1 file changed, 30 insertions(+), 31 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md index b0d978e6..b53697f8 100644 --- a/src/sec_certs/data/reference_annotations/readme.md +++ b/src/sec_certs/data/reference_annotations/readme.md @@ -1,17 +1,39 @@ # Reference annotations -This folder contains data and the methodology (presented below) related to learning the reference annotations. +This folder contains data related to learning the reference annotations. This document also describeds the utilized methodology. - The folder [split](split) contains split of the CC Dataset to `train/valid/test` splits for learning. -- The csv file [manually_annotated_references.csv](./manually_annotated_references.csv) contains manually acquired labels to references obtained with the methodology outlined below. -- The folder `adam` contains manual annotations created by Adam -- The folder `jano` contains manual annotations created by Jano -- The folder `conflicts` contains conflicting annotations between Adam and Jano, as weel as their resolution -- The contents of the `final` folder can thus be obtained by taking annotations either from `adam` or `jano` folder and masking them by `resolution_label` from `conflicts` folder. +- The csv file [outdated_manually_annotated_references.csv](./outdated_manually_annotated_references.csv) contains manually acquired labels to references obtained with **old methodology** for the sake of paper 1 submission. +- The folder [adam](adam/) contains manual annotations created by Adam +- The folder [jano](jano/) contains manual annotations created by Jano +- The folder [conflicts](conflicts/) contains conflicting annotations between Adam and Jano, as weel as their resolution +- The contents of the [final](final/) folder can thus be obtained by taking annotations either from `adam` or `jano` folder and masking them by `resolution_label` from `conflicts` folder. + +## Reference classification methodology + +### Data splits and manual annotations + +1. Two co-authors independently inspect identical set of 100 random certificates and capture the observed relations into reference taxonomy to form the annotation guidelines. See [reference taxonomy](#reference-taxonomy) below. +2. We split all certificates for which we register a direct outgoing reference in either security target or certification report into `train/valid/test` splits in `30/20/50` fashion (see [split](split/)). +3. We sample 100 train, 100 valid, 200 test pairs of reference instances (represented by `(dgst, canonical_reference_keyword)` pairs) for manual annotations. +4. Two co-authors independently assign each of these instances with a single label from the reference taxonomy. +5. We measure the inter-annotator agreement with Cohen's Kappa and percentage, see [inter-annotator agreement](#inter-annotator-agreement). +6. We resolve conflicts in the annotations in a meeting held by the co-authors. We use this consensual annotations for training and evaluation described below. + +### Supervised learning of the annotations + +1. For each pair `(dgst, referenced_cert_id)`, we recover the relevant text segments both from certification report and security target that mention the `referenced_cert_id`. +2. We apply text processing on the segments (e.g., unify re-certification vs. recertification, etc.) +3. We train a baseline model based on TF-IDF (or count vectorization in general), random forest and a soft-voting layer on top of that. + - Random forest classifies single segment with a probability of a given label. + - Soft voting compares probabilities of the given labels on all segments, takes their square and chooses the maximum. +4. We train a sentence transformer with the same soft-voting layer on top of that. +5. Finetune hyperparameters. +6. We evalute the results on the test set using weighted F1 score. ### Reference taxonomy -After manually inspecting random certificates, we have identified the following reference meanings: +After manually inspecting ~100 random certificates, we have identified the following reference meanings: - **Component used**: The referenced certificate is a component used in the examined certificate (e.g., IC used by a smartcard). Some evaluation results were likely shared/re-used. - **Component shared**: The referenced certificate shares some components with the examined certificate. Some evaluation results were likely shared/re-used. @@ -27,30 +49,7 @@ These can be further merged into the following super-categories: - **Previous version**: `previous_version` and `recertification` - **None**: `None` or `irrelevant` -### Reference classification methodology - -**Data splits and manual annotations**: - -1. Two authors inspect random certificates (~100) and capture the observed relations into reference taxonomy -2. Split all certificates for which we register a direct outgoing reference in either security target or certification report into `train/valid/test` splits in `30/20/50` fashion (see [split](split/)). -3. Sample 100 train, 100 valid, 200 test pairs of `(dgst, canonical_reference_keyword)` for manual annotations. -4. Two co-authors independently assign each of these pairs with a single label from the reference taxonomy. -5. Measure the inter-annotator agreement with Cohen's Kappa. -6. Resolve conflicts in the annotations in a meeting held by the co-authors. Use this consensual annotations for training and evaluation described below. - -**Learning the annotations**: - -1. For each pair `(dgst, referenced_cert_id)`, recover the relevant segments both from certification report and security target that mention the `referenced_cert_id` -2. Apply text processing on the segments (e.g., unify re-certification vs. recertification, etc.) -3. Train a baseline model based on TF-IDF (or count vectorization in general), random forest, and a soft-voting layer on top of that. - - Random forest classifies single segment to a probability of a given label - - Soft voting compares probabilities of the given labels on all segments, takes their square and chooses the maximum. -4. Train a sentence transformer with the same soft-voting layer on top of that. -5. Finetune hyperparameters. -6. Evaluate on test set. - - -## Inter-annotator agreement +### Inter-annotator agreement The inter-annotator agreement is measured both with Cohen's Kappa and with percentage. The results are as follows: -- cgit v1.3.1 From d21539ea3b6e06d3e0ba4705c740c6ef5d85e895 Mon Sep 17 00:00:00 2001 From: Adam Janovsky Date: Fri, 20 Oct 2023 10:26:49 +0200 Subject: rename recertification: re-evaluation --- .../data/reference_annotations/adam/test.csv | 60 +++++++++++----------- .../data/reference_annotations/adam/train.csv | 32 ++++++------ .../data/reference_annotations/adam/valid.csv | 38 +++++++------- .../data/reference_annotations/conflicts/test.csv | 4 +- .../data/reference_annotations/conflicts/train.csv | 4 +- .../data/reference_annotations/conflicts/valid.csv | 2 +- .../data/reference_annotations/final/test.csv | 60 +++++++++++----------- .../data/reference_annotations/final/train.csv | 32 ++++++------ .../data/reference_annotations/final/valid.csv | 38 +++++++------- .../data/reference_annotations/jano/test.csv | 56 ++++++++++---------- .../data/reference_annotations/jano/train.csv | 30 +++++------ .../data/reference_annotations/jano/valid.csv | 36 ++++++------- src/sec_certs/data/reference_annotations/readme.md | 4 +- 13 files changed, 198 insertions(+), 198 deletions(-) (limited to 'src/sec_certs/data/reference_annotations') diff --git a/src/sec_certs/data/reference_annotations/adam/test.csv b/src/sec_certs/data/reference_annotations/adam/test.csv index 19f4d790..e9702c4f 100644 --- a/src/sec_certs/data/reference_annotations/adam/test.csv +++ b/src/sec_certs/data/reference_annotations/adam/test.csv @@ -1,14 +1,14 @@ dgst,canonical_reference_keyword,label,comment -15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,recertification,None -15b121492722bffb,BSI-DSZ-CC-0754-2012,recertification,None -16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,recertification,None +15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,re-evaluation,None +15b121492722bffb,BSI-DSZ-CC-0754-2012,re-evaluation,None +16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,re-evaluation,None 16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None 16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None 173097995b7a7f12,ANSSI-CC-2012/49,component_shared,None 173097995b7a7f12,ANSSI-CC-2013/28,evaluation_reused,None 173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None 173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None -177dee4c1051e612,BSI-DSZ-CC-0854-2013,recertification,None +177dee4c1051e612,BSI-DSZ-CC-0854-2013,re-evaluation,None 17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None 183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None 183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None @@ -16,21 +16,21 @@ dgst,canonical_reference_keyword,label,comment 18b7e5bdf459ca13,NSCIB-CC-07-09219,component_used,None 195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None 195366bfba7213a5,BSI-DSZ-CC-0322-2005,component_used,None -195366bfba7213a5,BSI-DSZ-CC-0399-2007,recertification,None +195366bfba7213a5,BSI-DSZ-CC-0399-2007,re-evaluation,None 1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None 19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None -1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,recertification,None +1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,re-evaluation,None 1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None 1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None 1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None 1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None -1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,recertification,None +1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,re-evaluation,None 1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,None 1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,evaluation_reused,None -1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,recertification,None +1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,re-evaluation,None 1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None -1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,recertification,None -2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,recertification,None +1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,re-evaluation,None +2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,re-evaluation,None 21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None 21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None 22ede463dbf1a105,ANSSI-CC-2015/36,evaluation_reused,None @@ -41,7 +41,7 @@ dgst,canonical_reference_keyword,label,comment 23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,None 24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None 24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None -259df98e7476a843,BSI-DSZ-CC-0666-2012,recertification,None +259df98e7476a843,BSI-DSZ-CC-0666-2012,re-evaluation,None 259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None 25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None 25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None @@ -55,20 +55,20 @@ dgst,canonical_reference_keyword,label,comment 296618f10b019d9c,BSI-DSZ-CC-0417-2008,evaluation_reused,None 29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None 29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None -29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,recertification,None +29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,re-evaluation,None 2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None 2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None -2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,recertification,None +2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,re-evaluation,None 2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None 2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None 2bd9661657e78278,ANSSI-CC-2013/13,component_used,None -2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,recertification,None -2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,recertification,None -2d046178aa118fff,BSI-DSZ-CC-0293-2005,recertification,None +2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,re-evaluation,None +2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,re-evaluation,None +2d046178aa118fff,BSI-DSZ-CC-0293-2005,re-evaluation,None 2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None 2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None 2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None -2f9feaf4121720da,BSI-DSZ-CC-0547-2009,recertification,None +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,re-evaluation,None 30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None 3183669bf78db43b,ANSSI-CC-2015/45,component_used,None 3183669bf78db43b,ANSSI-CC-2016/64,component_used,None @@ -76,12 +76,12 @@ dgst,canonical_reference_keyword,label,comment 31fd07069aebf013,BSI-DSZ-CC-0466-2008,evaluation_reused,None 328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None 328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None -3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,recertification,None +3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,re-evaluation,None 3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None 337ece90615ed69d,ANSSI-CC-2010/02,component_used,None 3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None 33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None -33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,recertification,None +33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,re-evaluation,None 34506b8fab6dd7c3,ANSSI-CC-2014/22,evaluation_reused,None 34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None 34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None @@ -125,9 +125,9 @@ dgst,canonical_reference_keyword,label,comment 3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None 384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,None,unclear 384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None -3976c9e492193315,BSI-DSZ-CC-1113-2021,recertification,None +3976c9e492193315,BSI-DSZ-CC-1113-2021,re-evaluation,None 39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None -3a3b0a9113835307,BSI-DSZ-CC-0402-2008,recertification,None +3a3b0a9113835307,BSI-DSZ-CC-0402-2008,re-evaluation,None 3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None 3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None 3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None @@ -146,15 +146,15 @@ dgst,canonical_reference_keyword,label,comment 3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None 3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None 3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None -3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,recertification,None -3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,recertification,None +3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,re-evaluation,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,re-evaluation,None 3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None 3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None -3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,recertification,None +3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,re-evaluation,None 3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None 3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None 4057f5dbe4ffa4cd,ANSSI-CC-2017/57,evaluation_reused,None -40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,recertification,None +40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,re-evaluation,None 40fa0a4cb5193977,ANSSI-CC-2018/27,evaluation_reused,None 40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None 40fc6ad0aed92913,ANSSI-CC-2017/54,evaluation_reused,None @@ -163,7 +163,7 @@ dgst,canonical_reference_keyword,label,comment 41bd2924e9afced5,ANSSI-CC-2010/50,evaluation_reused,None 420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None 424f6d496746258b,ANSSI-CC-2019/12,component_used,None -424f6d496746258b,BSI-DSZ-CC-0995-2018,recertification,None +424f6d496746258b,BSI-DSZ-CC-0995-2018,re-evaluation,None 4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None 42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None 42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,None @@ -182,14 +182,14 @@ dgst,canonical_reference_keyword,label,comment 4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,previous_version,None 47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None 4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None -4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,recertification,None +4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,re-evaluation,None 4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None 4893140ae5daadaf,CRP278,component_used,None -48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,recertification,None +48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,re-evaluation,None 48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None 48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None 491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None -491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,recertification,None +491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,re-evaluation,None 49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None 49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None 49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None @@ -198,4 +198,4 @@ dgst,canonical_reference_keyword,label,comment 4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None 4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None 4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None -4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,recertification,None +4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,re-evaluation,None diff --git a/src/sec_certs/data/reference_annotations/adam/train.csv b/src/sec_certs/data/reference_annotations/adam/train.csv index 6da4fc05..1f74ca84 100644 --- a/src/sec_certs/data/reference_annotations/adam/train.csv +++ b/src/sec_certs/data/reference_annotations/adam/train.csv @@ -4,8 +4,8 @@ e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None 8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None 680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None -3515801dee00995f,BSI-DSZ-CC-0447-2008,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 -6a999675c9422dfb,BSI-DSZ-CC-0353-2006,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +3515801dee00995f,BSI-DSZ-CC-0447-2008,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0447-2008 +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0353-2006 12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert 921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned @@ -13,13 +13,13 @@ d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation wh c175537338906951,ANSSI-CC-2016/70,component_shared,None fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None -f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0863-2013 cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None dba4b5a166f9456a,ANSSI-CC-2014/76,evaluation_reused,None aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None b42009eb34746731,ANSSI-CC-2021/50,previous_version,maybe previous_version 463ecd64b7506048,ANSSI-CC-2018/52,component_used,None -c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,recertification,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,re-evaluation,None c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None 0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None 2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None @@ -31,16 +31,16 @@ af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None 54273dd266fce692,ANSSI-CC-2015/15,component_used,None 4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None -d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,recertification,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,re-evaluation,None a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None 8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,unclear 371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 -e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,recertification,None +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,re-evaluation,None 4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None -a867d281d34d34b7,BSI-DSZ-CC-0976-2015,recertification,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,re-evaluation,None 16bdbde359584f99,ANSSI-CC-2017/61,component_used,None 34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None @@ -59,9 +59,9 @@ f869956d14ea1694,ANSSI-CC-2012/68,component_used,None aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None 9529eb793550093c,ANSSI-CC-2018/40,previous_version,None -7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,recertification,None -ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,recertification,None -ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,recertification,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,re-evaluation,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,re-evaluation,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,re-evaluation,None ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None 30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None 6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None @@ -70,18 +70,18 @@ ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None 055985699aee1e09,ANSSI-CC-2010/02,component_used,None b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None 691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None -a07548a64fac5794,BSI-DSZ-CC-0350-2007,recertification,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,re-evaluation,None 19c60923e31777e7,ANSSI-CC-2017/49,component_used,None f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None -42799014c183e1b3,BSI-DSZ-CC-0429-2007,recertification,None -3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,recertification,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,re-evaluation,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,re-evaluation,None 5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear 4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None 30120e4f3aa2f30a,BSI-DSZ-CC-0897-V2-2014,component_used,None -c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,recertification,None +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,"unclear, mentions that this is a “re-evaluation” but not a “re-certification”" 97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None @@ -89,11 +89,11 @@ bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,"unclear, mentions th df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None 6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None 1b8705a0486b3be1,NSCIB-CC-0209053-CR,previous_version,None -a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,recertification,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,re-evaluation,None cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None 27d11629261d8806,ANSSI-CC-2014/59,component_used,None a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,None -d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,recertification,None +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,re-evaluation,None c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery" 84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None 36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None diff --git a/src/sec_certs/data/reference_annotations/adam/valid.csv b/src/sec_certs/data/reference_annotations/adam/valid.csv index 948845eb..26954167 100644 --- a/src/sec_certs/data/reference_annotations/adam/valid.csv +++ b/src/sec_certs/data/reference_annotations/adam/valid.csv @@ -1,9 +1,9 @@ dgst,canonical_reference_keyword,label,comment a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None -657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,recertification,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,re-evaluation,None 0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None -983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,recertification,None -8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,recertification,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,re-evaluation,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,re-evaluation,None 774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,None c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,evaluation_reused,None a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None @@ -11,11 +11,11 @@ a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None 24de96ea505af909,ANSSI-CC-2021/29,component_used,None 2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None -b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,recertification,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,re-evaluation,None d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None 4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,None 22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None -2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,recertification,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,re-evaluation,None 6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None 120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None 994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None @@ -23,24 +23,24 @@ d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None f05de4db911ec8af,CRP243,previous_version,None 38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None -fc875e0381a28435,BSI-DSZ-CC-0432-2007,recertification,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,re-evaluation,None c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None 2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_used,None 2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None 833ff87d69cff5db,383-4-184,None,unclear c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None -647d17d44745a532,BSI-DSZ-CC-0904-2015,recertification,None -4489bfc781a82281,BSI-DSZ-CC-0817-2013,recertification,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,re-evaluation,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,re-evaluation,None 94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None 37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None 5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None -5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,recertification,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,re-evaluation,None d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None 0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,unclear 84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None -1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,recertification,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,re-evaluation,None d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None -38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,recertification,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,re-evaluation,None e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None 4b3f577896c80f8a,ANSSI-CC-2016/44,previous_version,None @@ -56,7 +56,7 @@ f6317f134f532696,ANSSI-CC-2017/24,component_shared,None 1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None 2532fe8c7214c485,BSI-DSZ-CC-0633-2010,evaluation_reused,None b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None -35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,recertification,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,re-evaluation,None edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,component_shared,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the @@ -67,17 +67,17 @@ process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. Th TOE itself. The Security Target was updated. " 3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_shared,None 0fe8df0e85116b61,ANSSI-CC-2010/33,component_shared,None -bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,recertification,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,re-evaluation,None b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,None -7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,recertification,None -81273108dd167b98,BSI-DSZ-CC-0523-2008,recertification,None +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,re-evaluation,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,re-evaluation,None 63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None 4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None f4d510a39278e687,ANSSI-CC-2018/32,component_used,None c23bb542201aa511,ANSSI-CC-2014/14,component_shared,None 625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None -cae968d4926648bd,BSI-DSZ-CC-0754-2012,recertification,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,re-evaluation,None 000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None 66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None @@ -86,16 +86,16 @@ a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None 69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None 50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None -5cf51622fafaf868,BSI-DSZ-CC-1005-2016,recertification,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,re-evaluation,None 1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None 0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None 8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None 6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None 4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None -03fad65e5af65088,BSI-DSZ-CC-0696-2011,recertification,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,re-evaluation,None 720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None -9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,recertification,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None 73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None 2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None diff --git a/src/sec_certs/data/reference_annotations/conflicts/test.csv b/src/sec_certs/data/reference_annotations/conflicts/test.csv index dca3c452..20701c17 100644 --- a/src/sec_certs/data/reference_annotations/conflicts/test.csv +++ b/src/sec_certs/data/reference_annotations/conflicts/test.csv @@ -4,7 +4,7 @@ dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label 18b7e5bdf459ca13,NSCIB-CC-07-09219,component_used,previous_version,previous_version 195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None,component_used 195366bfba7213a5,BSI-DSZ-CC-0322-2005,component_used,None,previous_version -195366bfba7213a5,BSI-DSZ-CC-0399-2007,recertification,None,recertification +195366bfba7213a5,BSI-DSZ-CC-0399-2007,re-evaluation,None,re-evaluation 1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None,component_used 1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,evaluation_reused,component_shared,component_shared 22ede463dbf1a105,ANSSI-CC-2015/36,evaluation_reused,component_used,component_used @@ -12,7 +12,7 @@ dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label 26c4912b140fde9f,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,component_used,component_used 27ccec0740bb7915,ANSSI-CC-2010/35,evaluation_reused,component_shared,component_shared 296618f10b019d9c,BSI-DSZ-CC-0417-2008,evaluation_reused,component_used,component_used -2f9feaf4121720da,BSI-DSZ-CC-0547-2009,recertification,evaluation_reused,recertification +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,re-evaluation,evaluation_reused,re-evaluation 31fd07069aebf013,BSI-DSZ-CC-0466-2008,evaluation_reused,component_used,previous_version 34506b8fab6dd7c3,ANSSI-CC-2014/22,evaluation_reused,previous_version,previous_version 054bd8196e015197,ANSSI-CC-2011/07,None,evaluation_reused,evaluation_reused diff --git a/src/sec_certs/data/reference_annotations/conflicts/train.csv b/src/sec_certs/data/reference_annotations/conflicts/train.csv index 8ac397ce..18ed3252 100644 --- a/src/sec_certs/data/reference_annotations/conflicts/train.csv +++ b/src/sec_certs/data/reference_annotations/conflicts/train.csv @@ -6,5 +6,5 @@ a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,evaluation_reused,component b42009eb34746731,ANSSI-CC-2021/50,previous_version,evaluation_reused,previous_version bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,None,evaluation_reused c175537338906951,ANSSI-CC-2016/70,component_shared,component_used,component_shared -c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,recertification,evaluation_reused,evaluation_reused -dba4b5a166f9456a,ANSSI-CC-2014/76,evaluation_reused,previous_version,recertification +c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,re-evaluation,evaluation_reused,evaluation_reused +dba4b5a166f9456a,ANSSI-CC-2014/76,evaluation_reused,previous_version,re-evaluation diff --git a/src/sec_certs/data/reference_annotations/conflicts/valid.csv b/src/sec_certs/data/reference_annotations/conflicts/valid.csv index b18f88ee..ba016b70 100644 --- a/src/sec_certs/data/reference_annotations/conflicts/valid.csv +++ b/src/sec_certs/data/reference_annotations/conflicts/valid.csv @@ -8,7 +8,7 @@ dgst,canonical_reference_keyword,label_adam,label_jano,resolution_label 4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,component_used,evaluation_reused 4b3f577896c80f8a,ANSSI-CC-2016/44,previous_version,component_used,evaluation_reused 774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,component_used,evaluation_reused -7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,recertification,previous_version,recertification +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,re-evaluation,previous_version,re-evaluation b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,evaluation_reused,previous_version c23bb542201aa511,ANSSI-CC-2014/14,component_shared,component_used,previous_version c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,evaluation_reused,component_shared,None diff --git a/src/sec_certs/data/reference_annotations/final/test.csv b/src/sec_certs/data/reference_annotations/final/test.csv index fbda9fd8..28695033 100644 --- a/src/sec_certs/data/reference_annotations/final/test.csv +++ b/src/sec_certs/data/reference_annotations/final/test.csv @@ -1,14 +1,14 @@ dgst,canonical_reference_keyword,label,comment -15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,recertification,None -15b121492722bffb,BSI-DSZ-CC-0754-2012,recertification,None -16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,recertification,None +15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,re-evaluation,None +15b121492722bffb,BSI-DSZ-CC-0754-2012,re-evaluation,None +16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,re-evaluation,None 16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None 16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None 173097995b7a7f12,ANSSI-CC-2012/49,component_shared,None 173097995b7a7f12,ANSSI-CC-2013/28,component_used,None 173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None 173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None -177dee4c1051e612,BSI-DSZ-CC-0854-2013,recertification,None +177dee4c1051e612,BSI-DSZ-CC-0854-2013,re-evaluation,None 17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None 183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None 183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None @@ -16,21 +16,21 @@ dgst,canonical_reference_keyword,label,comment 18b7e5bdf459ca13,NSCIB-CC-07-09219,previous_version,None 195366bfba7213a5,BSI-DSZ-CC-0266-2005,component_used,None 195366bfba7213a5,BSI-DSZ-CC-0322-2005,previous_version,None -195366bfba7213a5,BSI-DSZ-CC-0399-2007,recertification,None +195366bfba7213a5,BSI-DSZ-CC-0399-2007,re-evaluation,None 1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,component_used,None 19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None -1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,recertification,None +1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,re-evaluation,None 1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None 1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None 1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None 1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None -1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,recertification,None +1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,re-evaluation,None 1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,None 1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,component_shared,None -1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,recertification,None +1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,re-evaluation,None 1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None -1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,recertification,None -2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,recertification,None +1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,re-evaluation,None +2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,re-evaluation,None 21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None 21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None 22ede463dbf1a105,ANSSI-CC-2015/36,component_used,None @@ -41,7 +41,7 @@ dgst,canonical_reference_keyword,label,comment 23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,None 24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None 24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None -259df98e7476a843,BSI-DSZ-CC-0666-2012,recertification,None +259df98e7476a843,BSI-DSZ-CC-0666-2012,re-evaluation,None 259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None 25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None 25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None @@ -55,20 +55,20 @@ dgst,canonical_reference_keyword,label,comment 296618f10b019d9c,BSI-DSZ-CC-0417-2008,component_used,None 29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None 29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None -29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,recertification,None +29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,re-evaluation,None 2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None 2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None -2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,recertification,None +2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,re-evaluation,None 2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None 2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None 2bd9661657e78278,ANSSI-CC-2013/13,component_used,None -2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,recertification,None -2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,recertification,None -2d046178aa118fff,BSI-DSZ-CC-0293-2005,recertification,None +2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,re-evaluation,None +2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,re-evaluation,None +2d046178aa118fff,BSI-DSZ-CC-0293-2005,re-evaluation,None 2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None 2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None 2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None -2f9feaf4121720da,BSI-DSZ-CC-0547-2009,recertification,None +2f9feaf4121720da,BSI-DSZ-CC-0547-2009,re-evaluation,None 30e9ce0969941ffb,BSI-DSZ-CC-0782-V2-2015,component_used,None 3183669bf78db43b,ANSSI-CC-2015/45,component_used,None 3183669bf78db43b,ANSSI-CC-2016/64,component_used,None @@ -76,12 +76,12 @@ dgst,canonical_reference_keyword,label,comment 31fd07069aebf013,BSI-DSZ-CC-0466-2008,previous_version,None 328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None 328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None -3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,recertification,None +3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,re-evaluation,None 3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None 337ece90615ed69d,ANSSI-CC-2010/02,component_used,None 3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None 33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None -33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,recertification,None +33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,re-evaluation,None 34506b8fab6dd7c3,ANSSI-CC-2014/22,previous_version,None 34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None 34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None @@ -125,9 +125,9 @@ dgst,canonical_reference_keyword,label,comment 3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None 384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,component_used,unclear 384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None -3976c9e492193315,BSI-DSZ-CC-1113-2021,recertification,None +3976c9e492193315,BSI-DSZ-CC-1113-2021,re-evaluation,None 39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None -3a3b0a9113835307,BSI-DSZ-CC-0402-2008,recertification,None +3a3b0a9113835307,BSI-DSZ-CC-0402-2008,re-evaluation,None 3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None 3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None 3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None @@ -146,15 +146,15 @@ dgst,canonical_reference_keyword,label,comment 3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None 3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None 3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None -3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,recertification,None -3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,recertification,None +3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,re-evaluation,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,re-evaluation,None 3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None 3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None -3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,recertification,None +3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,re-evaluation,None 3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None 3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None 4057f5dbe4ffa4cd,ANSSI-CC-2017/57,previous_version,None -40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,recertification,None +40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,re-evaluation,None 40fa0a4cb5193977,ANSSI-CC-2018/27,component_shared,None 40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None 40fc6ad0aed92913,ANSSI-CC-2017/54,previous_version,None @@ -163,7 +163,7 @@ dgst,canonical_reference_keyword,label,comment 41bd2924e9afced5,ANSSI-CC-2010/50,previous_version,None 420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None 424f6d496746258b,ANSSI-CC-2019/12,component_used,None -424f6d496746258b,BSI-DSZ-CC-0995-2018,recertification,None +424f6d496746258b,BSI-DSZ-CC-0995-2018,re-evaluation,None 4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None 42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None 42d2e68c29d9eef7,ANSSI-CC-2018/30,evaluation_reused,None @@ -182,14 +182,14 @@ dgst,canonical_reference_keyword,label,comment 4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,irrelevant,None 47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None 4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None -4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,recertification,None +4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,re-evaluation,None 4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None 4893140ae5daadaf,CRP278,component_used,None -48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,recertification,None +48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,re-evaluation,None 48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None 48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None 491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None -491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,recertification,None +491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,re-evaluation,None 49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None 49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None 49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None @@ -198,4 +198,4 @@ dgst,canonical_reference_keyword,label,comment 4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None 4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None 4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None -4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,recertification,None +4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,re-evaluation,None diff --git a/src/sec_certs/data/reference_annotations/final/train.csv b/src/sec_certs/data/reference_annotations/final/train.csv index b6b81d30..6e106fc2 100644 --- a/src/sec_certs/data/reference_annotations/final/train.csv +++ b/src/sec_certs/data/reference_annotations/final/train.csv @@ -24,12 +24,12 @@ dgst,canonical_reference_keyword,label,comment 30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None 317713948b4473b3,BSI-DSZ-CC-0798-2012,component_shared,None 34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None -3515801dee00995f,BSI-DSZ-CC-0447-2008,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 +3515801dee00995f,BSI-DSZ-CC-0447-2008,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0447-2008 36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None 371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None -3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,recertification,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,re-evaluation,None 42799014c183e1b3,BSI-DSZ-CC-0312-2005,component_used,None -42799014c183e1b3,BSI-DSZ-CC-0429-2007,recertification,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,re-evaluation,None 463ecd64b7506048,ANSSI-CC-2018/52,component_used,None 4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None 4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None @@ -41,11 +41,11 @@ dgst,canonical_reference_keyword,label,comment 627bfd69c2a831cf,ANSSI-CC-2014/37,evaluation_reused,None 680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None 691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None -6a999675c9422dfb,BSI-DSZ-CC-0353-2006,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0353-2006 6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None 6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None 744a7a202d909323,ANSSI-CC-2017/24,component_used,None -7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,recertification,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,re-evaluation,None 845bb039719ac5d8,ANSSI-CC-2010/40,previous_version,None 84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None 8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None @@ -53,12 +53,12 @@ dgst,canonical_reference_keyword,label,comment 921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned 9529eb793550093c,ANSSI-CC-2018/40,previous_version,None 97d00a9f198a6e57,ANSSI-CC-2015/66,component_used,None -a07548a64fac5794,BSI-DSZ-CC-0350-2007,recertification,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,re-evaluation,None a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,component_used,None -a867d281d34d34b7,BSI-DSZ-CC-0976-2015,recertification,None -a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,recertification,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,re-evaluation,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,re-evaluation,None aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None ab3af998dff7a2ef,BSI-DSZ-CC-0973-V2-2016,component_used,None @@ -70,30 +70,30 @@ bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,evaluation_reused,"unclear, mentions th c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None c175537338906951,ANSSI-CC-2016/70,component_shared,None c32378a010479b33,BSI-DSZ-CC-0891-V2-2016,evaluation_reused,None -c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,recertification,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,re-evaluation,None c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert -d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,recertification,None +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,re-evaluation,None d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None -d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,recertification,None -dba4b5a166f9456a,ANSSI-CC-2014/76,recertification,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,re-evaluation,None +dba4b5a166f9456a,ANSSI-CC-2014/76,re-evaluation,None df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None -e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,recertification,None +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,re-evaluation,None e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified M7892 G12 component e6e8add5e4db2d9d,BSI-DSZ-CC-0891-V4-2019,component_used,None -ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,recertification,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,re-evaluation,None ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None ecba01acba8df2ec,BSI-DSZ-CC-0633-2010,component_used,None -ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,recertification,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,re-evaluation,None ee1c6dd97918d74a,ANSSI-CC-2012/71,evaluation_reused,None f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None -f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0863-2013 f869956d14ea1694,ANSSI-CC-2012/68,component_used,None fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,irrelevant,None diff --git a/src/sec_certs/data/reference_annotations/final/valid.csv b/src/sec_certs/data/reference_annotations/final/valid.csv index a74d7a45..2d662981 100644 --- a/src/sec_certs/data/reference_annotations/final/valid.csv +++ b/src/sec_certs/data/reference_annotations/final/valid.csv @@ -1,9 +1,9 @@ dgst,canonical_reference_keyword,label,comment a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None -657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,recertification,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,re-evaluation,None 0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None -983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,recertification,None -8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,recertification,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,re-evaluation,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,re-evaluation,None 774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,evaluation_reused,None c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,None,None a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None @@ -11,11 +11,11 @@ a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None 24de96ea505af909,ANSSI-CC-2021/29,component_used,None 2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None -b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,recertification,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,re-evaluation,None d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None 4b3f577896c80f8a,ANSSI-CC-2015/80,evaluation_reused,None 22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None -2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,recertification,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,re-evaluation,None 6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None 120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None 994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None @@ -23,24 +23,24 @@ d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None f05de4db911ec8af,CRP243,previous_version,None 38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None -fc875e0381a28435,BSI-DSZ-CC-0432-2007,recertification,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,re-evaluation,None c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None 2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_shared,None 2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None 833ff87d69cff5db,383-4-184,None,unclear c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None -647d17d44745a532,BSI-DSZ-CC-0904-2015,recertification,None -4489bfc781a82281,BSI-DSZ-CC-0817-2013,recertification,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,re-evaluation,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,re-evaluation,None 94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None 37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None 5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None -5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,recertification,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,re-evaluation,None d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None 0f53257eb74bee54,BSI-DSZ-CC-0417-2008,component_used,unclear 84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None -1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,recertification,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,re-evaluation,None d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None -38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,recertification,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,re-evaluation,None e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None 4b3f577896c80f8a,ANSSI-CC-2016/44,evaluation_reused,None @@ -56,7 +56,7 @@ f6317f134f532696,ANSSI-CC-2017/24,previous_version,None 1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None 2532fe8c7214c485,BSI-DSZ-CC-0633-2010,component_used,None b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None -35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,recertification,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,re-evaluation,None edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the @@ -67,17 +67,17 @@ process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. Th TOE itself. The Security Target was updated. " 3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_used,None 0fe8df0e85116b61,ANSSI-CC-2010/33,previous_version,None -bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,recertification,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,re-evaluation,None b00bb452e7a8176c,ANSSI-CC-2016/57,previous_version,None -7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,recertification,None -81273108dd167b98,BSI-DSZ-CC-0523-2008,recertification,None +7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,re-evaluation,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,re-evaluation,None 63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None 4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None f4d510a39278e687,ANSSI-CC-2018/32,component_used,None c23bb542201aa511,ANSSI-CC-2014/14,previous_version,None 625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None -cae968d4926648bd,BSI-DSZ-CC-0754-2012,recertification,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,re-evaluation,None 000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None 66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None @@ -86,16 +86,16 @@ a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None 69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None 50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None -5cf51622fafaf868,BSI-DSZ-CC-1005-2016,recertification,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,re-evaluation,None 1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None 0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None 8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None 6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None 4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None -03fad65e5af65088,BSI-DSZ-CC-0696-2011,recertification,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,re-evaluation,None 720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None -9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,recertification,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None 73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None 2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None diff --git a/src/sec_certs/data/reference_annotations/jano/test.csv b/src/sec_certs/data/reference_annotations/jano/test.csv index b241171b..52e72176 100644 --- a/src/sec_certs/data/reference_annotations/jano/test.csv +++ b/src/sec_certs/data/reference_annotations/jano/test.csv @@ -1,14 +1,14 @@ dgst,canonical_reference_keyword,label,comment -15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,recertification,None -15b121492722bffb,BSI-DSZ-CC-0754-2012,recertification,None -16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,recertification,evaluation_reused more like +15ae64b85b3e28d7,BSI-DSZ-CC-0394-2006,re-evaluation,None +15b121492722bffb,BSI-DSZ-CC-0754-2012,re-evaluation,None +16513a1bff79b46a,BSI-DSZ-CC-1044-V2-2019,re-evaluation,evaluation_reused more like 16b0a0811bc4fe82,BSI-DSZ-CC-0169-2002,component_used,None 16b24ff1bf3c079b,ANSSI-CC-2012/22,component_used,None 173097995b7a7f12,ANSSI-CC-2012/49,evaluation_reused,None 173097995b7a7f12,ANSSI-CC-2013/28,component_used,None 173704f0d2b8a02f,ANSSI-CC-2018/08,component_used,None 173704f0d2b8a02f,ANSSI-CC-2018/19,component_used,None -177dee4c1051e612,BSI-DSZ-CC-0854-2013,recertification,None +177dee4c1051e612,BSI-DSZ-CC-0854-2013,re-evaluation,None 17bfc3d0570ab3d0,ANSSI-CC-2016/09,component_used,None 183a5cd3e2aaa0ec,ANSSI-CC-2017/76,component_used,None 183a5cd3e2aaa0ec,BSI-DSZ-CC-0891-V2-2016,component_used,None @@ -19,18 +19,18 @@ dgst,canonical_reference_keyword,label,comment 195366bfba7213a5,BSI-DSZ-CC-0399-2007,None,None 1999d4ed82b3188e,BSI-DSZ-CC-0451-2007,None,None 19d3185c40be008d,ANSSI-CC-2016/65,evaluation_reused,None -1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,recertification,None +1ac8de53c0894d18,BSI-DSZ-CC-0814-2012,re-evaluation,None 1b2764a62ffe86a2,BSI-DSZ-CC-0978-2016,component_used,None 1bcfaefe46abccf0,BSI-DSZ-CC-0917-2014,component_used,None 1c8567a1b1a6c12a,BSI-DSZ-CC-1040-2019,component_used,None 1cc05dbb992431b9,ANSSI-CC-2021/29,component_used,None -1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,recertification,None +1d1df0fb541e49b8,BSI-DSZ-CC-1020-V2-2017,re-evaluation,None 1d9ae732c5dec242,ANSSI-CC-2009/37,component_used,also previous version to a degree 1ea31bb5f6a15995,BSI-DSZ-CC-0808-V3-2017,component_shared,None -1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,recertification,None +1ea31bb5f6a15995,BSI-DSZ-CC-0809-V2-2016,re-evaluation,None 1ea31bb5f6a15995,BSI-DSZ-CC-0978-V2-2017,component_used,None -1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,recertification,None -2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,recertification,None +1ff08f79cb89c1de,BSI-DSZ-CC-0939-2015,re-evaluation,None +2078424be58e4db1,OCSI/CERT/SYS/08/2017/RC,re-evaluation,None 21951e191e55b66e,BSI-DSZ-CC-0782-2012,component_used,None 21cffadbbb87c205,BSI-DSZ-CC-0348-2006,component_used,None 22ede463dbf1a105,ANSSI-CC-2015/36,component_used,None @@ -41,7 +41,7 @@ dgst,canonical_reference_keyword,label,comment 23a85a7f9c07412f,ANSSI-CC-2016/33,evaluation_reused,component_used probably 24624a4e60ddedd4,BSI-DSZ-CC-0868-2014,previous_version,None 24624a4e60ddedd4,BSI-DSZ-CC-0951-2015,component_used,None -259df98e7476a843,BSI-DSZ-CC-0666-2012,recertification,None +259df98e7476a843,BSI-DSZ-CC-0666-2012,re-evaluation,None 259e0ab1c4b690cf,ANSSI-CC-2012/70,component_used,None 25c07c5d0a05d86c,ANSSI-CC-2015/08,component_shared,None 25c07c5d0a05d86c,ANSSI-CC-2015/15,component_used,None @@ -55,16 +55,16 @@ dgst,canonical_reference_keyword,label,comment 296618f10b019d9c,BSI-DSZ-CC-0417-2008,component_used,None 29cc495628e76ffa,BSI-DSZ-CC-0845-2012,component_used,None 29f2dd90f57ea948,BSI-DSZ-CC-0837-V2-2014,component_used,None -29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,recertification,None +29fb07d8f74e734b,BSI-DSZ-CC-0303-2006,re-evaluation,None 2a07b85f61ac08b0,ANSSI-CC-2019/59,evaluation_reused,None 2a91f42389fda90d,BSI-DSZ-CC-0782-V2-2015,component_used,None -2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,recertification,None +2adb03f5294c31c2,BSI-DSZ-CC-0608-2010,re-evaluation,None 2b0342d5b70b7a78,ANSSI-CC-2010/03,component_used,None 2bd9661657e78278,ANSSI-CC-2012/39,previous_version,None 2bd9661657e78278,ANSSI-CC-2013/13,component_used,None -2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,recertification,None -2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,recertification,None -2d046178aa118fff,BSI-DSZ-CC-0293-2005,recertification,None +2be10f342e68a89e,BSI-DSZ-CC-0963-V2-2017,re-evaluation,None +2c41f9dfcd5014e6,BSI-DSZ-CC-0973-2016,re-evaluation,None +2d046178aa118fff,BSI-DSZ-CC-0293-2005,re-evaluation,None 2d0b4100c3ead88a,BSI-DSZ-CC-0945-V2-2018,component_used,None 2eee0fdd5cdaf565,ANSSI-CC-2017/07,component_used,None 2eee0fdd5cdaf565,BSI-DSZ-CC-0891-V2-2016,component_used,None @@ -76,12 +76,12 @@ dgst,canonical_reference_keyword,label,comment 31fd07069aebf013,BSI-DSZ-CC-0466-2008,component_used,None 328fee1e52f7ac82,BSI-DSZ-CC-0410-2007,component_used,None 328fee1e52f7ac82,BSI-DSZ-CC-0680-2010,component_used,None -3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,recertification,None +3334d22b6559d0e2,BSI-DSZ-CC-0944-2014,re-evaluation,None 3334d22b6559d0e2,BSI-DSZ-CC-0963-V2-2017,component_used,None 337ece90615ed69d,ANSSI-CC-2010/02,component_used,None 3392558f04e04663,BSI-DSZ-CC-0782-V2-2015,component_used,None 33cada8c95bf55c5,BSI-DSZ-CC-0879-V3-2018,component_used,None -33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,recertification,None +33cada8c95bf55c5,BSI-DSZ-CC-1028-2017,re-evaluation,None 34506b8fab6dd7c3,ANSSI-CC-2014/22,previous_version,None 34ed76f0c32d9e93,ANSSI-CC-2012/49,evaluation_reused,None 34ed76f0c32d9e93,ANSSI-CC-2013/18,component_used,None @@ -125,9 +125,9 @@ dgst,canonical_reference_keyword,label,comment 3817c0aca007989e,BSI-DSZ-CC-0626-2009,component_used,None 384dca297bd4f1ca,BSI-DSZ-CC-0837-V2-2014,component_used,None 384dca297bd4f1ca,BSI-DSZ-CC-0978-2016,component_used,None -3976c9e492193315,BSI-DSZ-CC-1113-2021,recertification,None +3976c9e492193315,BSI-DSZ-CC-1113-2021,re-evaluation,None 39c07ebbca541145,BSI-DSZ-CC-0917-2014,component_used,None -3a3b0a9113835307,BSI-DSZ-CC-0402-2008,recertification,None +3a3b0a9113835307,BSI-DSZ-CC-0402-2008,re-evaluation,None 3a3b0a9113835307,BSI-DSZ-CC-0468-2007,component_used,None 3a3b0a9113835307,CCEVS-VR-07-0054,component_used,None 3a3b0a9113835307,CCEVS-VR-VID10271-2007,component_used,None @@ -146,15 +146,15 @@ dgst,canonical_reference_keyword,label,comment 3ea02a62856c752a,ANSSI-CC-2012/70,component_used,None 3ea02a62856c752a,ANSSI-CC-2012/72,component_used,None 3ea02a62856c752a,BSI-DSZ-CC-0719-2011,component_used,None -3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,recertification,None -3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,recertification,None +3f188bbf2af01b25,OCSI/CERT/IMQ/07/2017/RC,re-evaluation,None +3f22bd3eaef5d64d,BSI-DSZ-CC-0957-2015,re-evaluation,None 3f22bd3eaef5d64d,BSI-DSZ-CC-0978-2016,component_used,None 3f4b6e4f245f6fab,ANSSI-CC-2017/49,component_used,None -3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,recertification,None +3f8475de6ea558ac,BSI-DSZ-CC-0427-2007,re-evaluation,None 3feda0b8b5637540,BSI-DSZ-CC-1059-V4-2021,component_used,None 3feda0b8b5637540,NSCIB-CC-67206-CR5,component_used,None 4057f5dbe4ffa4cd,ANSSI-CC-2017/57,previous_version,None -40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,recertification,None +40f62cd468a546cc,OCSI/CERT/ATS/05/2020/RC,re-evaluation,None 40fa0a4cb5193977,ANSSI-CC-2018/27,previous_version,None 40fa0a4cb5193977,BSI-DSZ-CC-0879-V3-2018,component_used,None 40fc6ad0aed92913,ANSSI-CC-2017/54,component_used,None @@ -163,7 +163,7 @@ dgst,canonical_reference_keyword,label,comment 41bd2924e9afced5,ANSSI-CC-2010/50,previous_version,None 420876ec1e5ba657,ANSSI-CC-2012/30,component_used,None 424f6d496746258b,ANSSI-CC-2019/12,component_used,None -424f6d496746258b,BSI-DSZ-CC-0995-2018,recertification,None +424f6d496746258b,BSI-DSZ-CC-0995-2018,re-evaluation,None 4285d9b580f8a2a6,ANSSI-CC-2013/59,component_used,None 42bc15eb26cf2eec,BSI-DSZ-CC-0829-2012,component_used,None 42d2e68c29d9eef7,ANSSI-CC-2018/30,previous_version,None @@ -182,14 +182,14 @@ dgst,canonical_reference_keyword,label,comment 4750f5114dcaa60d,BSI-DSZ-CC-0891-V4-2019,irrelevant,BSI-DSZ-CC-0891-V4-2018 mentioned in report 47913a485c3c8a18,ANSSI-CC-2019/28,component_used,None 4832a44c0df0bad2,ANSSI-CC-2019/12,component_used,None -4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,recertification,None +4832a44c0df0bad2,BSI-DSZ-CC-1074-2019,re-evaluation,None 4893140ae5daadaf,BSI-DSZ-CC-0782-2012,component_used,None 4893140ae5daadaf,CRP278,component_shared,None -48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,recertification,None +48a41f3f5110db1f,BSI-DSZ-CC-0350-2007,re-evaluation,None 48d897f17754c7ce,BSI-DSZ-CC-0978-2016,component_used,None 48fd3d84b1a0bc68,BSI-DSZ-CC-0348-2006,component_used,None 491c766e35ae9f6e,BSI-DSZ-CC-0879-V4-2020,component_used,None -491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,recertification,None +491c766e35ae9f6e,BSI-DSZ-CC-1071-V3-2020,re-evaluation,None 49e6665ee56d3bfc,ANSSI-CC-2015/45,component_used,None 49e6665ee56d3bfc,ANSSI-CC-2016/64,component_used,None 49e6665ee56d3bfc,BSI-DSZ-CC-0782-V2-2015,component_used,None @@ -198,4 +198,4 @@ dgst,canonical_reference_keyword,label,comment 4abc83fee59586b2,NSCIB-CC-200270-CR,previous_version,None 4b2f963b48e0f954,ANSSI-CC-2010/11,component_used,None 4b5521f85ab3fff7,BSI-DSZ-CC-0879-V4-2020,component_used,None -4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,recertification,None +4b5521f85ab3fff7,BSI-DSZ-CC-1071-V4-2020,re-evaluation,None diff --git a/src/sec_certs/data/reference_annotations/jano/train.csv b/src/sec_certs/data/reference_annotations/jano/train.csv index 18dce096..cce5f69f 100644 --- a/src/sec_certs/data/reference_annotations/jano/train.csv +++ b/src/sec_certs/data/reference_annotations/jano/train.csv @@ -4,8 +4,8 @@ e676143c80802a59,BSI-DSZ-CC-0891-V4-2019,component_used,the previously certified ab3af998dff7a2ef,ANSSI-CC-2017/47,component_used,None 8ba22f6c9651edc3,ANSSI-CC-2013/55,component_used,None 680aeb0a20a9fed3,BSI-DSZ-CC-0870-2014,component_used,None -3515801dee00995f,BSI-DSZ-CC-0447-2008,recertification,This is a re-certification based on BSI-DSZ-CC-0447-2008 -6a999675c9422dfb,BSI-DSZ-CC-0353-2006,recertification,This is a re-certification based on BSI-DSZ-CC-0353-2006 +3515801dee00995f,BSI-DSZ-CC-0447-2008,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0447-2008 +6a999675c9422dfb,BSI-DSZ-CC-0353-2006,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0353-2006 12e20466a0f08342,ANSSI-CC-2016/79,evaluation_reused,unclear whether component used d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation where referenced cert is mentioned as recertification of some other cert 921e042759b30033,BSI-DSZ-CC-0978-2016,component_used,composition scheme “[COMP]” mentioned @@ -13,13 +13,13 @@ d2a4b8cb9ae7fe8f,BSI-DSZ-CC-1110-V4-2021,component_used,complicated situation wh c175537338906951,ANSSI-CC-2016/70,component_used,None fb9ec9e846ea4e87,ANSSI-CC-2017/47,component_used,None ecba01acba8df2ec,BSI-DSZ-CC-0555-2009,component_used,None -f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,recertification,This is a re-certification based on BSI-DSZ-CC-0863-2013 +f71fca5fc684df8b,BSI-DSZ-CC-0863-2013,re-evaluation,This is a re-certification based on BSI-DSZ-CC-0863-2013 cacdfdafce47678c,ANSSI-CC-2014/59,component_used,None dba4b5a166f9456a,ANSSI-CC-2014/76,previous_version,None aa096ffde94b953b,BSI-DSZ-CC-0411-2007,component_used,None b42009eb34746731,ANSSI-CC-2021/50,evaluation_reused,maybe previous_version 463ecd64b7506048,ANSSI-CC-2018/52,component_used,None -c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,recertification,None +c5a60dbdb668bc10,BSI-DSZ-CC-0891-2015,re-evaluation,None c96343baf608174d,BSI-DSZ-CC-0645-2010,component_used,None 0b3e1cdf3ef9413d,BSI-DSZ-CC-0837-V2-2014,component_used,None 2c15dfc106ebf8d8,ANSSI-CC-2012/24,component_used,None @@ -31,16 +31,16 @@ af61a31e3fd0d6f0,ANSSI-CC-2010/02,evaluation_reused,None d49988efd778ca9d,ANSSI-CC-2010/02,component_used,None 54273dd266fce692,ANSSI-CC-2015/15,component_used,None 4bede5d2af4cb11f,ANSSI-CC-2019/59,evaluation_reused,None -d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,recertification,None +d75282fdda80b6fd,BSI-DSZ-CC-0609-2010,re-evaluation,None a69ec271f8651d1b,BSI-DSZ-CC-1040-2019,component_used,None f6579adcbf5faa99,BSI-DSZ-CC-1040-2019,component_used,None 8bdb610131555c12,BSI-DSZ-CC-0645-2010,component_used,None b3b7ac7aae87793d,BSI-DSZ-CC-0945-V2-2018,None,unclear 371dca18821f7714,BSI-DSZ-CC-0945-2017,component_used,None f17fe9ceea628f62,BSI-DSZ-CC-0911-2014,component_shared,The Basic Access Control mechanism was subject of the evaluation process BSI-DSZ-CC-0911-2014 -e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,recertification,None +e0b122da55f1f002,BSI-DSZ-CC-0961-V2-2018,re-evaluation,None 4f5f41ecf7517e63,ANSSI-CC-2016/80,evaluation_reused,None -a867d281d34d34b7,BSI-DSZ-CC-0976-2015,recertification,None +a867d281d34d34b7,BSI-DSZ-CC-0976-2015,re-evaluation,None 16bdbde359584f99,ANSSI-CC-2017/61,component_used,None 34370b67b5e675c3,ANSSI-CC-2013/47,component_used,None a6fe8fe0aaf2fa92,ANSSI-CC-2018/26,previous_version,None @@ -59,9 +59,9 @@ f869956d14ea1694,ANSSI-CC-2012/68,component_used,None aea11fd4d2a7709d,BSI-DSZ-CC-0809-V3-2017,component_shared,The further security mechanism Basic Access Control is subject of the separate evaluation process BSI-DSZ-CC-0809-V3-2017 05726637bd47a762,BSI-DSZ-CC-0891-V3-2018,component_used,None 9529eb793550093c,ANSSI-CC-2018/40,evaluation_reused,None -7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,recertification,None -ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,recertification,None -ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,recertification,None +7a8d4ed693d443d7,BSI-DSZ-CC-1128-V3-2021,re-evaluation,None +ed22d4c0f09c3e3a,BSI-DSZ-CC-0719-2011,re-evaluation,None +ea6d47427a2bb349,BSI-DSZ-CC-0782-2012,re-evaluation,None ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None 30f71b100c5cebae,ANSSI-CC-2015/15,component_used,None 6eb6b29c45f6355f,BSI-DSZ-CC-0782-V4-2018,component_used,None @@ -70,12 +70,12 @@ ead1076787fdd7e4,BSI-DSZ-CC-0399-2007,component_used,None 055985699aee1e09,ANSSI-CC-2010/02,component_used,None b61363e51ed90c7a,ANSSI-CC-2019/12,component_used,None 691119fe8a2f8e1c,BSI-DSZ-CC-0963-2015,component_used,None -a07548a64fac5794,BSI-DSZ-CC-0350-2007,recertification,None +a07548a64fac5794,BSI-DSZ-CC-0350-2007,re-evaluation,None 19c60923e31777e7,ANSSI-CC-2017/49,component_used,None f17fe9ceea628f62,BSI-DSZ-CC-0750-V2-2014,component_used,None c96343baf608174d,BSI-DSZ-CC-0750-2011,component_used,None -42799014c183e1b3,BSI-DSZ-CC-0429-2007,recertification,None -3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,recertification,None +42799014c183e1b3,BSI-DSZ-CC-0429-2007,re-evaluation,None +3f45ee333ce457bb,BSI-DSZ-CC-1002-2018,re-evaluation,None 5f4b39feb2a82dac,BSI-DSZ-CC-0266-2005,component_used,unclear 4a6100f1ddaef93b,BSI-DSZ-CC-0410-2007,component_used,None c06d86db280f9b2b,BSI-DSZ-CC-0437-2008,component_used,None @@ -89,11 +89,11 @@ bab97726875c0f14,BSI-DSZ-CC-0957-V2-2016,None,"unclear, mentions that this is a df6728b8420ab3b4,BSI-DSZ-CC-0827-V4-2016,component_used,None 6f8d7a6a1dea6a3a,ANSSI-CC-2019/28,component_used,None 1b8705a0486b3be1,NSCIB-CC-0209053-CR,evaluation_reused,None -a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,recertification,None +a9d336b90e8b94a2,BSI-DSZ-CC-0410-2007,re-evaluation,None cfdacd53c732343c,BSI-DSZ-CC-1110-V2-2019,component_used,None 27d11629261d8806,ANSSI-CC-2014/59,component_used,None a76d1c0d9964d583,BSI-DSZ-CC-0417-2008,evaluation_reused,None -d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,recertification,None +d2b0cb5a911b8ef8,BSI-DSZ-CC-0729-2011,re-evaluation,None c38734a9eeea0ff8,BSI-DSZ-CC-0950-V2-2018,None,"unclear, mentions just ToE delivery" 84a75b6fc33b669e,ANSSI-CC-2014/08,evaluation_reused,None 36ed04f4b45e3ab9,BSI-DSZ-CC-0555-2009,component_used,None diff --git a/src/sec_certs/data/reference_annotations/jano/valid.csv b/src/sec_certs/data/reference_annotations/jano/valid.csv index a55055f6..197ff250 100644 --- a/src/sec_certs/data/reference_annotations/jano/valid.csv +++ b/src/sec_certs/data/reference_annotations/jano/valid.csv @@ -1,9 +1,9 @@ dgst,canonical_reference_keyword,label,comment a21d17fd2d8b8edc,BSI-DSZ-CC-0813-2012,component_used,None -657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,recertification,None +657f8d0cc39bbcdc,BSI-DSZ-CC-0946-2014,re-evaluation,None 0efe2627f1ce8ac2,BSI-DSZ-CC-0351-2006,component_used,None -983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,recertification,None -8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,recertification,None +983d16512ae92d46,BSI-DSZ-CC-0891-V3-2018,re-evaluation,None +8dd850c81b49a1f9,BSI-DSZ-CC-0697-2011,re-evaluation,None 774cb0f28f7900f9,BSI-DSZ-CC-0232-2004,component_used,None c37a9b7ae53093d8,CCEVS-VR-VID10256-2012,component_shared,None a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None @@ -11,11 +11,11 @@ a5adb726852f5cc5,ANSSI-CC-2009/37,component_used,None cc533c087f06ad0d,BSI-DSZ-CC-1040-2019,component_used,None 24de96ea505af909,ANSSI-CC-2021/29,component_used,None 2793414918738c7f,BSI-DSZ-CC-0857-2013,component_used,None -b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,recertification,None +b402cb61be362e5f,BSI-DSZ-CC-0946-V2-2015,re-evaluation,None d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None 4b3f577896c80f8a,ANSSI-CC-2015/80,component_used,None 22388445fe620ac0,ANSSI-CC-2014/20,evaluation_reused,None -2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,recertification,None +2c2244c35d126bfb,BSI-DSZ-CC-0794-2011,re-evaluation,None 6a85bb1d21a35e7d,BSI-DSZ-CC-0227-2004,component_used,None 120a49c0aa284f68,BSI-DSZ-CC-0813-2012,component_used,None 994ac7d986a99e50,BSI-DSZ-CC-0640-2010,component_used,None @@ -23,24 +23,24 @@ d96438a7165e5d82,BSI-DSZ-CC-0410-2007,component_used,None f05de4db911ec8af,CRP243,component_used,None 38f79f1a48e7844f,ANSSI-CC-2017/47,component_used,None fe55e99f4f3dc195,ANSSI-CC-2013/33,evaluation_reused,None -fc875e0381a28435,BSI-DSZ-CC-0432-2007,recertification,None +fc875e0381a28435,BSI-DSZ-CC-0432-2007,re-evaluation,None c23bb542201aa511,BSI-DSZ-CC-0829-2012,component_used,None 2532fe8c7214c485,BSI-DSZ-CC-0675-2011,component_used,None 2cbc543a7c7de8b5,BSI-DSZ-CC-0813-2012,component_used,None 833ff87d69cff5db,383-4-184,None,unclear c7c1215819127a1c,BSI-DSZ-CC-0322-2005,component_used,None -647d17d44745a532,BSI-DSZ-CC-0904-2015,recertification,None -4489bfc781a82281,BSI-DSZ-CC-0817-2013,recertification,None +647d17d44745a532,BSI-DSZ-CC-0904-2015,re-evaluation,None +4489bfc781a82281,BSI-DSZ-CC-0817-2013,re-evaluation,None 94fb86d31446e531,BSI-DSZ-CC-0958-2015,component_used,None 37bae9eaf342542b,BSI-DSZ-CC-0349-2006,evaluation_reused,None 5c06e4fb1887fdc7,BSI-DSZ-CC-0782-V2-2015,component_used,None -5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,recertification,None +5bf14a907f9a4c65,BSI-DSZ-CC-0527-2008,re-evaluation,None d314fa7ee445bd1d,BSI-DSZ-CC-0410-2007,component_used,None 0f53257eb74bee54,BSI-DSZ-CC-0417-2008,None,unclear 84d6a370e4f38ddc,ANSSI-CC-2019/12,component_used,None -1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,recertification,None +1a5f86cb1d942c37,BSI-DSZ-CC-1110-2019,re-evaluation,None d696a9daccdb2a9f,BSI-DSZ-CC-1136-2021,component_used,None -38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,recertification,None +38e7d62918a4f9b6,BSI-DSZ-CC-0688-2013,re-evaluation,None e6057ffc5085a192,ANSSI-CC-2014/48,evaluation_reused,None db7627e56b8621b1,BSI-DSZ-CC-0639-2010,component_used,None 4b3f577896c80f8a,ANSSI-CC-2016/44,component_used,None @@ -56,7 +56,7 @@ f6317f134f532696,ANSSI-CC-2017/24,evaluation_reused,None 1b20b1629ba7d3a6,ANSSI-CC-2012/72,component_used,None 2532fe8c7214c485,BSI-DSZ-CC-0633-2010,component_used,None b1ef468658a9a0c1,NSCIB-CC-0448219-CR,evaluation_reused,None -35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,recertification,None +35482ae218713d54,BSI-DSZ-CC-0951-V3-2018,re-evaluation,None edb3056f1838e6ae,BSI-DSZ-CC-0348-2006,evaluation_reused,"Quite weird, The evaluation was performed as a re-evaluation process based on BSI-DSZCC-0227-2004, BSI-DSZ-CC-0312-2005 and BSI-DSZ-CC-0348-2006. Compared to BSI-DSZ-CC-0227-2004 the TOE was re-evaluated because of technical and yield improvements. Compared to BSI-DSZ-CC-0348-2006 the @@ -67,17 +67,17 @@ process. For production sites, results from BSI-DSZ-CC-0312-2005 were reused. Th TOE itself. The Security Target was updated. " 3f2a6d4893c18634,NSCIB-CC-12-36243-CR2,component_used,None 0fe8df0e85116b61,ANSSI-CC-2010/33,evaluation_reused,None -bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,recertification,None +bbc41d7d09e40c0c,BSI-DSZ-CC-0404-2007,re-evaluation,None b00bb452e7a8176c,ANSSI-CC-2016/57,evaluation_reused,None 7e58bfc14edf68e4,OCSI/CERT/TEC/01/2013/RC,previous_version,None -81273108dd167b98,BSI-DSZ-CC-0523-2008,recertification,None +81273108dd167b98,BSI-DSZ-CC-0523-2008,re-evaluation,None 63c1045055ec3b58,BSI-DSZ-CC-0266-2005,component_used,None a3c99646acb0bb6f,BSI-DSZ-CC-0837-V2-2014,component_used,None 4c017087976d05eb,ANSSI-CC-2020/45,evaluation_reused,None f4d510a39278e687,ANSSI-CC-2018/32,component_used,None c23bb542201aa511,ANSSI-CC-2014/14,component_used,None 625b243397001f01,BSI-DSZ-CC-1059-2018,component_used,None -cae968d4926648bd,BSI-DSZ-CC-0754-2012,recertification,None +cae968d4926648bd,BSI-DSZ-CC-0754-2012,re-evaluation,None 000176284683faa2,BSI-DSZ-CC-0640-2010,component_used,None 66faa094a9333d1a,ANSSI-CC-2014/86,evaluation_reused,None a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None @@ -86,16 +86,16 @@ a6dc46792d4e41c8,BSI-DSZ-CC-0203-2003,component_used,None 69b15e884cc13c70,BSI-DSZ-CC-0891-V2-2016,component_used,None a4bbf07046109fc9,BSI-DSZ-CC-0935-2015,component_used,None 50aef770cb4f028f,BSI-DSZ-CC-1059-V3-2019,component_used,None -5cf51622fafaf868,BSI-DSZ-CC-1005-2016,recertification,None +5cf51622fafaf868,BSI-DSZ-CC-1005-2016,re-evaluation,None 1e23ae421d7a2d01,BSI-DSZ-CC-0633-V2-2014,component_used,None 0c7ef6c32cbdee47,ANSSI-CC-2017/61,component_used,None 8928e191ee2fd3b9,BSI-DSZ-CC-0523-2008,component_used,None 6ed3cd21c6a2c9d0,BSI-DSZ-CC-1107-2020,component_used,None 4962c0d8106b8b5e,ANSSI-CC-2015/37,evaluation_reused,None e6057ffc5085a192,BSI-DSZ-CC-0829-2012,component_used,None -03fad65e5af65088,BSI-DSZ-CC-0696-2011,recertification,None +03fad65e5af65088,BSI-DSZ-CC-0696-2011,re-evaluation,None 720aab4f782f4c60,BSI-DSZ-CC-0555-2009,component_used,None -9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,recertification,None +9664c0f0ec6401b9,BSI-DSZ-CC-0891-V2-2016,re-evaluation,None d55eb8c97d71e843,ANSSI-CC-2021/49,previous_version,None 73e0b884cd04d9bc,NSCIB-CC-12-36243,component_used,None 2793414918738c7f,BSI-DSZ-CC-0633-V2-2014,component_used,None diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md index b53697f8..b10a2e30 100644 --- a/src/sec_certs/data/reference_annotations/readme.md +++ b/src/sec_certs/data/reference_annotations/readme.md @@ -38,7 +38,7 @@ After manually inspecting ~100 random certificates, we have identified the follo - **Component used**: The referenced certificate is a component used in the examined certificate (e.g., IC used by a smartcard). Some evaluation results were likely shared/re-used. - **Component shared**: The referenced certificate shares some components with the examined certificate. Some evaluation results were likely shared/re-used. - **Evaluation reused**: The evaluation results of the referenced certificate were used for evaluation of the examined certificate, due to reasons that could not be resolved. -- **Recertification**: The examined certificate is a re-certification of the referenced certificate. +- **Re-evaluation**: The examined certificate is a re-evaluation of the referenced certificate. For definition of re-evaluation, see [Assurance Continuity: CCRA Requirements](https://www.commoncriteriaportal.org/files/operatingprocedures/CCDB-011-v2.2-2021-Sep-30-Final-Assurance_Continuity.pdf). - **Previous version**: The product in the referenced certificate is a previous version of the product in the examined certificate and the re-certification is not explicitly mentioned. - **None**: The annotator could not assign any of the previous contexts. - **Irrelevant**: The reference is irrelevant to the studied certificate (typo, left-out reference from a template, ...) @@ -46,7 +46,7 @@ After manually inspecting ~100 random certificates, we have identified the follo These can be further merged into the following super-categories: - **Some sub-component relationship** `component_used`, `component_shared`, and `evaluation_reused` -- **Previous version**: `previous_version` and `recertification` +- **Previous version**: `previous_version` and `re-evaluation` - **None**: `None` or `irrelevant` ### Inter-annotator agreement -- cgit v1.3.1 From 80190b01aeda844b9d3ea8684284130c44f1453e Mon Sep 17 00:00:00 2001 From: adamjanovsky Date: Tue, 14 Nov 2023 10:04:13 +0100 Subject: bump references --- .../annotator_agreement.ipynb | 98 ---- .../inter_annotator_agreement.ipynb | 72 +++ .../cc/reference_annotations/tf_idf_baseline.ipynb | 176 ------- pyproject.toml | 1 + requirements/all_requirements.txt | 110 +++- requirements/compile.sh | 9 +- requirements/dev_requirements.txt | 52 +- requirements/nlp_requirements.txt | 557 +++++++++++++++++++++ requirements/requirements.txt | 50 +- requirements/test_requirements.txt | 50 +- src/sec_certs/constants.py | 7 +- src/sec_certs/data/reference_annotations/readme.md | 31 +- src/sec_certs/dataset/cc.py | 69 +-- src/sec_certs/dataset/fips.py | 2 +- src/sec_certs/model/__init__.py | 5 +- src/sec_certs/model/reference_finder.py | 223 +++++++++ src/sec_certs/model/references/annotator.py | 126 ----- .../model/references/annotator_trainer.py | 164 ------ src/sec_certs/model/references/reference_finder.py | 223 --------- .../model/references/segment_extractor.py | 390 --------------- src/sec_certs/model/references_nlp/__init__.py | 13 + src/sec_certs/model/references_nlp/annotator.py | 101 ++++ .../model/references_nlp/annotator_trainer.py | 173 +++++++ src/sec_certs/model/references_nlp/evaluation.py | 90 ++++ .../model/references_nlp/feature_extraction.py | 542 ++++++++++++++++++++ .../model/references_nlp/segment_extractor.py | 354 +++++++++++++ src/sec_certs/model/references_nlp/training.py | 68 +++ 27 files changed, 2449 insertions(+), 1307 deletions(-) delete mode 100644 notebooks/cc/reference_annotations/annotator_agreement.ipynb create mode 100644 notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb delete mode 100644 notebooks/cc/reference_annotations/tf_idf_baseline.ipynb create mode 100644 requirements/nlp_requirements.txt create mode 100644 src/sec_certs/model/reference_finder.py delete mode 100644 src/sec_certs/model/references/annotator.py delete mode 100644 src/sec_certs/model/references/annotator_trainer.py delete mode 100644 src/sec_certs/model/references/reference_finder.py delete mode 100644 src/sec_certs/model/references/segment_extractor.py create mode 100644 src/sec_certs/model/references_nlp/__init__.py create mode 100644 src/sec_certs/model/references_nlp/annotator.py create mode 100644 src/sec_certs/model/references_nlp/annotator_trainer.py create mode 100644 src/sec_certs/model/references_nlp/evaluation.py create mode 100644 src/sec_certs/model/references_nlp/feature_extraction.py create mode 100644 src/sec_certs/model/references_nlp/segment_extractor.py create mode 100644 src/sec_certs/model/references_nlp/training.py (limited to 'src/sec_certs/data/reference_annotations') diff --git a/notebooks/cc/reference_annotations/annotator_agreement.ipynb b/notebooks/cc/reference_annotations/annotator_agreement.ipynb deleted file mode 100644 index 6b941323..00000000 --- a/notebooks/cc/reference_annotations/annotator_agreement.ipynb +++ /dev/null @@ -1,98 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from sklearn.metrics import cohen_kappa_score\n", - "from sec_certs.dataset import CCDataset\n", - "from pathlib import Path\n", - "\n", - "REPO_ROOT = Path(\"./../../../\").resolve()\n", - "REF_ANNOTATIONS_PATH = REPO_ROOT / \"dataset/reference_annotations\"\n", - "\n", - "def get_agreement_df(adam_df_path, jano_df_path):\n", - " df_adam = pd.read_csv(adam_df_path)\n", - " df_jano = pd.read_csv(jano_df_path)\n", - " df_jano.label = df_jano.label.map(lambda x: x.upper())\n", - " df = pd.merge(df_adam, df_jano, on=[\"dgst\", \"referenced_cert_id\"], suffixes=(\"_adam\", \"_jano\"))\n", - " df[\"agreement\"] = df.label_adam == df.label_jano\n", - " return df" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# Load dataframes with individual labels and create new `agreement` column\n", - "agreement_train = get_agreement_df(REF_ANNOTATIONS_PATH / \"adam_train.csv\", REF_ANNOTATIONS_PATH / \"jano_train.csv\")\n", - "agreement_valid = get_agreement_df(REF_ANNOTATIONS_PATH / \"adam_valid.csv\", REF_ANNOTATIONS_PATH / \"jano_valid.csv\")\n", - "agreement_all = pd.concat([agreement_train, agreement_valid])\n", - "conflicting_df = agreement_all.loc[~agreement_all.agreement].copy()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.7617784008411355" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Measure Cohen's kappa\n", - "# For interpretation, see https://hrcak.srce.hr/file/132393.\n", - "# We're aiming for 0.9+.\n", - "cohen_kappa_score(agreement_all.label_adam, agreement_all.label_jano)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "# Load CC dataset and fill-in paths to pdfs of conflicting instances\n", - "dset = CCDataset.from_json(REPO_ROOT / \"dataset/cc_final_run_may_23/dataset.json\")\n", - "conflicting_df[\"report_path\"] = conflicting_df.dgst.map(lambda x: \"file:/\" + str(dset[x].state.report_pdf_path))\n", - "conflicting_df[\"st_path\"] = conflicting_df.dgst.map(lambda x: \"file:/\" + str(dset[x].state.st_pdf_path))\n", - "conflicting_df.loc[:, [\"dgst\", \"referenced_cert_id\", \"label_adam\", \"label_jano\", \"comment\", \"report_path\", \"st_path\"]].to_csv(REF_ANNOTATIONS_PATH / \"conflicting.csv\", index=False)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.16" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb b/notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb new file mode 100644 index 00000000..d02a8975 --- /dev/null +++ b/notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cohen's Kappa: 0.7101271765978729\n", + "Percentage agreement: 0.8225\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "from pathlib import Path\n", + "from sklearn.metrics import cohen_kappa_score\n", + "\n", + "\n", + "def load_all_dataframes(base_folder: Path) -> pd.DataFrame:\n", + " splits = [\"train\", \"valid\", \"test\"]\n", + "\n", + " df_train, df_valid, df_test = pd.DataFrame(), pd.DataFrame(), pd.DataFrame()\n", + " for split in splits:\n", + " df = pd.read_csv(base_folder / f\"{split}.csv\")\n", + " if split == \"train\":\n", + " df_train = df\n", + " elif split == \"valid\":\n", + " df_valid = df\n", + " else:\n", + " df_test = df\n", + "\n", + " return pd.concat([df_train, df_valid, df_test])\n", + "\n", + "\n", + "REPO_ROOT = Path(\".\")\n", + "\n", + "adam_df = load_all_dataframes(REPO_ROOT / \"src/sec_certs/data/reference_annotations/adam\")\n", + "jano_df = load_all_dataframes(REPO_ROOT / \"src/sec_certs/data/reference_annotations/jano\")\n", + "agreement_series = adam_df.label == jano_df.label\n", + "\n", + "print(f\"Cohen's Kappa: {cohen_kappa_score(adam_df.label, jano_df.label)}\")\n", + "print(f\"Percentage agreement: {agreement_series.loc[agreement_series == True].count() / agreement_series.count()}\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/cc/reference_annotations/tf_idf_baseline.ipynb b/notebooks/cc/reference_annotations/tf_idf_baseline.ipynb deleted file mode 100644 index d7668a8a..00000000 --- a/notebooks/cc/reference_annotations/tf_idf_baseline.ipynb +++ /dev/null @@ -1,176 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import annotations\n", - "import os\n", - "\n", - "# When on Aura, it is important to first set CUDA_VISIBLE_DEVICES environment variable directly from notebook\n", - "# For available GPUs, see https://www.fi.muni.cz/tech/unix/aura.html.cs\n", - "# Must be done before any related package that leverages cuda is imported\n", - "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"MIG-56c53afb-6f08-5e5b-83fa-32fc6f09eeb0\"\n", - "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"FALSE\"\n", - "\n", - "import pandas as pd\n", - "from sec_certs.dataset import CCDataset\n", - "from pathlib import Path\n", - "from sec_certs.model.references.segment_extractor import ReferenceSegmentExtractor\n", - "from sklearn.metrics import ConfusionMatrixDisplay\n", - "from sec_certs.utils.helpers import compute_heuristics_version\n", - "from rapidfuzz import fuzz\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.metrics import classification_report\n", - "from sklearn.feature_extraction.text import TfidfVectorizer\n", - "import numpy as np\n", - "from sec_certs.utils.nlp import softmax\n", - "import torch\n", - "\n", - "\n", - "REPO_ROOT = Path(\".\").resolve()\n", - "DATASET_PATH = REPO_ROOT / \"dataset/cc_final_run_may_23/dataset.json\"\n", - "ANNOTATIONS_PATH = REPO_ROOT / \"src/sec_certs/data/reference_annotations/final/\"\n", - "\n", - "def replace_all(text: str, to_replce: set[str]) -> str:\n", - " for i in to_replce:\n", - " text = text.replace(i, \"\")\n", - " return text\n", - "\n", - "print(f\"GPU available: {torch.cuda.is_available()}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Compute name similarity" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_annotations = pd.read_csv(ANNOTATIONS_PATH / \"train.csv\")\n", - "valid_annotations = pd.read_csv(ANNOTATIONS_PATH / \"valid.csv\")\n", - "all_annotations = pd.concat([train_annotations, valid_annotations])\n", - "all_annotations = all_annotations[all_annotations.label != \"None\"].assign(label=lambda df: df.label.str.upper())\n", - "\n", - "dset = CCDataset.from_json(DATASET_PATH)\n", - "all_certs = {x.dgst: x for x in dset.certs.values()}\n", - "dset.certs = {x.dgst: x for x in dset.certs.values() if x.dgst in all_annotations.dgst.unique()}\n", - "\n", - "cert_id_to_name_mapping = {x.heuristics.cert_id: x.name for x in all_certs.values()}\n", - "all_annotations[\"referenced_cert_name\"] = all_annotations[\"referenced_cert_id\"].map(cert_id_to_name_mapping)\n", - "all_annotations[\"cert_name\"] = all_annotations[\"dgst\"].map(lambda x: dset[x].name)\n", - "all_annotations[\"cert_versions\"] = all_annotations[\"cert_name\"].map(compute_heuristics_version)\n", - "all_annotations = all_annotations.loc[all_annotations[\"referenced_cert_name\"].notnull()].copy()\n", - "all_annotations[\"referenced_cert_versions\"] = all_annotations[\"referenced_cert_name\"].map(compute_heuristics_version)\n", - "all_annotations[\"cert_name_stripped_version\"] = all_annotations.apply(lambda x: replace_all(x[\"cert_name\"], x[\"cert_versions\"]), axis=1)\n", - "all_annotations[\"referenced_cert_name_stripped_version\"] = all_annotations.apply(lambda x: replace_all(x[\"referenced_cert_name\"], x[\"referenced_cert_versions\"]), axis=1)\n", - "all_annotations[\"name_similarity\"] = all_annotations.apply(lambda x: fuzz.token_set_ratio(x[\"cert_name\"], x[\"referenced_cert_name\"]), axis=1)\n", - "all_annotations[\"name_similarity_stripped_version\"] = all_annotations.apply(lambda x: fuzz.token_set_ratio(x[\"cert_name_stripped_version\"], x[\"referenced_cert_name_stripped_version\"]), axis=1)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Retrieve segments" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = ReferenceSegmentExtractor()(dset.certs.values())\n", - "df = df.loc[df.label.notnull()].copy()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train TF-IDF\n", - "\n", - "(Train on individual segments, then agregate the results with sum of probabilities)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vectorizer = TfidfVectorizer()\n", - "df_train_exploded = df.loc[df.split == \"train\"].explode(\"segments\")\n", - "df_valid_exploded = df.loc[df.split == \"valid\"].explode(\"segments\")\n", - "\n", - "x_train = df_train_exploded[\"segments\"]\n", - "y_train = df_train_exploded[\"label\"]\n", - "\n", - "x_valid = df_valid_exploded[\"segments\"]\n", - "y_valid = df_valid_exploded[\"label\"]\n", - "\n", - "x_train_tfidf = vectorizer.fit_transform(x_train)\n", - "x_valid_tfidf = vectorizer.transform(x_valid)\n", - "\n", - "clf = RandomForestClassifier(n_estimators=100, random_state=42)\n", - "clf.fit(x_train_tfidf, y_train)\n", - "y_pred = clf.predict(x_valid_tfidf)\n", - "\n", - "df_train_exploded[\"y_proba\"] = clf.predict_proba(x_train_tfidf).tolist()\n", - "df_valid_exploded[\"y_proba\"] = clf.predict_proba(x_valid_tfidf).tolist()\n", - "\n", - "# Get mapping of labels to indices of the clf random forest classifier\n", - "label_mapping = {i: label for i, label in enumerate(clf.classes_)}\n", - "\n", - "# Now merge the segments back together and compute final prediction using the softmax of the sum of the probabilities\n", - "df_valid_retrieved = df_valid_exploded.loc[:, [\"dgst\", \"referenced_cert_id\", \"y_proba\"]].groupby([\"dgst\", \"referenced_cert_id\"]).agg(list).reset_index()\n", - "df_train_retrieved = df_train_exploded.loc[:, [\"dgst\", \"referenced_cert_id\", \"y_proba\"]].groupby([\"dgst\", \"referenced_cert_id\"]).agg(list).reset_index()\n", - "\n", - "def aggregate_results(x):\n", - " # Return the argmax of the sum of the probabilities obtained from the predictions on the individual segments\n", - " return label_mapping[int(np.argmax(softmax(np.power(x, 2).sum(axis=0))))]\n", - "\n", - "df_valid_retrieved[\"y_pred\"] = df_valid_retrieved[\"y_proba\"].map(aggregate_results)\n", - "df_train_retrieved[\"y_pred\"] = df_train_retrieved[\"y_proba\"].map(aggregate_results)\n", - "df_predictions = pd.concat([df_train_retrieved, df_valid_retrieved])\n", - "df_final = df.merge(df_predictions, on=[\"dgst\", \"referenced_cert_id\"])\n", - "\n", - "# Finally print the classification report on the aggregated results\n", - "print(classification_report(df_final.loc[df_final.split == \"valid\"].label, df_final.loc[df_final.split == \"valid\"].y_pred, zero_division=0))\n", - "\n", - "ConfusionMatrixDisplay.from_predictions(df_final.loc[df_final.split == \"valid\"].label, df_final.loc[df_final.split == \"valid\"].y_pred, xticks_rotation=90)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.1" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/pyproject.toml b/pyproject.toml index 8a271bcb..429d17da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ "ipython!=8.7.0", ] test = ["pytest", "coverage", "pytest-cov"] + nlp = ["catboost", "optuna", "setfit", "umap-learn", "plotly", "sklearn"] [project.urls] Homepage = "https://seccerts.org" diff --git a/requirements/all_requirements.txt b/requirements/all_requirements.txt index a0815502..d07b62b8 100644 --- a/requirements/all_requirements.txt +++ b/requirements/all_requirements.txt @@ -8,10 +8,8 @@ aiosignal==1.3.1 # via aiohttp alabaster==0.7.13 # via sphinx -appnope==0.1.3 - # via - # ipykernel - # ipython +alembic==1.12.1 + # via optuna asttokens==2.2.1 # via stack-data async-timeout==4.0.2 @@ -43,6 +41,8 @@ catalogue==2.0.8 # spacy # srsly # thinc +catboost==1.2.2 + # via sec-certs (./../pyproject.toml) certifi==2022.12.7 # via requests cfgv==3.3.1 @@ -59,6 +59,10 @@ click==8.1.3 # pip-tools # sec-certs (./../pyproject.toml) # typer +cmake==3.27.7 + # via triton +colorlog==6.7.0 + # via optuna comm==0.1.2 # via ipykernel confection==0.0.4 @@ -112,6 +116,7 @@ filelock==3.10.0 # huggingface-hub # torch # transformers + # triton # virtualenv fonttools==4.39.2 # via matplotlib @@ -123,8 +128,11 @@ fsspec[http]==2023.3.0 # via # datasets # evaluate + # fsspec gprof2dot==2022.7.29 # via pytest-profiling +graphviz==0.20.1 + # via catboost greenlet==2.0.2 # via sqlalchemy html5lib==1.1 @@ -173,6 +181,7 @@ jinja2==3.1.2 joblib==1.2.0 # via # nltk + # pynndescent # scikit-learn jsonschema==4.17.3 # via @@ -197,18 +206,29 @@ langcodes==3.3.0 # via spacy langdetect==1.0.9 # via sec-certs (./../pyproject.toml) +lit==17.0.3 + # via triton +llvmlite==0.41.1 + # via + # numba + # pynndescent lxml==4.9.2 # via # pikepdf # sec-certs (./../pyproject.toml) +mako==1.3.0 + # via alembic markdown-it-py==2.2.0 # via # mdit-py-plugins # myst-parser markupsafe==2.1.2 - # via jinja2 + # via + # jinja2 + # mako matplotlib==3.7.1 # via + # catboost # pysankeybeta # seaborn # sec-certs (./../pyproject.toml) @@ -268,13 +288,20 @@ nltk==3.8.1 # via sentence-transformers nodeenv==1.7.0 # via pre-commit +numba==0.58.1 + # via + # pynndescent + # umap-learn numpy==1.24.2 # via # blis + # catboost # contourpy # datasets # evaluate # matplotlib + # numba + # optuna # pandas # pyarrow # pysankeybeta @@ -288,6 +315,34 @@ numpy==1.24.2 # thinc # torchvision # transformers + # umap-learn +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.7.101 + # via torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +nvidia-cufft-cu11==10.9.0.58 + # via torch +nvidia-curand-cu11==10.2.10.91 + # via torch +nvidia-cusolver-cu11==11.4.0.1 + # via torch +nvidia-cusparse-cu11==11.7.4.91 + # via torch +nvidia-nccl-cu11==2.14.3 + # via torch +nvidia-nvtx-cu11==11.7.91 + # via torch +optuna==3.4.0 + # via sec-certs (./../pyproject.toml) packaging==23.0 # via # black @@ -298,7 +353,9 @@ packaging==23.0 # huggingface-hub # ipykernel # matplotlib + # optuna # pikepdf + # plotly # pydata-sphinx-theme # pytest # setuptools-scm @@ -308,6 +365,7 @@ packaging==23.0 # transformers pandas==1.5.3 # via + # catboost # datasets # evaluate # pysankeybeta @@ -343,6 +401,10 @@ platformdirs==3.1.1 # black # jupyter-core # virtualenv +plotly==5.18.0 + # via + # catboost + # sec-certs (./../pyproject.toml) pluggy==1.0.0 # via pytest pre-commit==3.1.1 @@ -380,10 +442,14 @@ pygments==2.14.0 # ipython # pydata-sphinx-theme # sphinx +pynndescent==0.5.10 + # via umap-learn pyparsing==3.0.9 # via matplotlib pypdf[crypto]==3.5.2 - # via sec-certs (./../pyproject.toml) + # via + # pypdf + # sec-certs (./../pyproject.toml) pyproject-hooks==1.0.0 # via build pyrsistent==0.19.3 @@ -417,6 +483,7 @@ pyyaml==6.0 # jupyter-cache # myst-nb # myst-parser + # optuna # pre-commit # sec-certs (./../pyproject.toml) # transformers @@ -451,13 +518,18 @@ ruff==0.0.239 # via sec-certs (./../pyproject.toml) scikit-learn==1.2.2 # via + # pynndescent # sec-certs (./../pyproject.toml) # sentence-transformers + # umap-learn scipy==1.10.1 # via + # catboost + # pynndescent # scikit-learn # sec-certs (./../pyproject.toml) # sentence-transformers + # umap-learn seaborn==0.12.2 # via # pysankeybeta @@ -473,10 +545,13 @@ setuptools-scm==7.1.0 six==1.16.0 # via # asttokens + # catboost # html5lib # langdetect # pytest-profiling # python-dateutil +sklearn==0.0.post11 + # via sec-certs (./../pyproject.toml) smart-open==6.3.0 # via # pathy @@ -519,7 +594,10 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sqlalchemy==1.4.46 - # via jupyter-cache + # via + # alembic + # jupyter-cache + # optuna srsly==2.4.6 # via # confection @@ -533,6 +611,10 @@ tabula-py==2.7.0 # via sec-certs (./../pyproject.toml) tabulate==0.9.0 # via jupyter-cache +tbb==2021.10.0 + # via umap-learn +tenacity==8.2.3 + # via plotly thinc==8.1.9 # via spacy threadpoolctl==3.1.0 @@ -543,6 +625,7 @@ torch==2.0.0 # via # sentence-transformers # torchvision + # triton torchvision==0.15.1 # via sentence-transformers tornado==6.2 @@ -555,10 +638,12 @@ tqdm==4.65.0 # evaluate # huggingface-hub # nltk + # optuna # sec-certs (./../pyproject.toml) # sentence-transformers # spacy # transformers + # umap-learn traitlets==5.9.0 # via # comm @@ -572,6 +657,8 @@ traitlets==5.9.0 # nbformat transformers==4.27.1 # via sentence-transformers +triton==2.0.0 + # via torch typer==0.7.0 # via # pathy @@ -586,6 +673,7 @@ types-urllib3==1.26.25.8 # via types-requests typing-extensions==4.5.0 # via + # alembic # huggingface-hub # mypy # myst-nb @@ -593,6 +681,8 @@ typing-extensions==4.5.0 # pydantic # setuptools-scm # torch +umap-learn==0.5.4 + # via sec-certs (./../pyproject.toml) urllib3==1.26.15 # via # requests @@ -609,6 +699,12 @@ webencodings==0.5.1 # via html5lib wheel==0.40.0 # via + # nvidia-cublas-cu11 + # nvidia-cuda-cupti-cu11 + # nvidia-cuda-runtime-cu11 + # nvidia-curand-cu11 + # nvidia-cusparse-cu11 + # nvidia-nvtx-cu11 # pip-tools # pytest-monitor widgetsnbextension==4.0.5 diff --git a/requirements/compile.sh b/requirements/compile.sh index c07386aa..b5b7703f 100755 --- a/requirements/compile.sh +++ b/requirements/compile.sh @@ -2,7 +2,8 @@ # See CONTRIBUTING.md for description -pip-compile --no-header --resolver=backtracking --output-file=requirements.txt ./../pyproject.toml -pip-compile --no-header --resolver=backtracking --extra dev -o dev_requirements.txt ./../pyproject.toml -pip-compile --no-header --resolver=backtracking --extra test -o test_requirements.txt ./../pyproject.toml -pip-compile --no-header --resolver=backtracking --extra dev --extra test -o all_requirements.txt ./../pyproject.toml +pip-compile --no-header --output-file=requirements.txt ./../pyproject.toml +pip-compile --no-header --extra dev -o dev_requirements.txt ./../pyproject.toml +pip-compile --no-header --extra test -o test_requirements.txt ./../pyproject.toml +pip-compile --no-header --extra nlp -o nlp_requirements.txt ./../pyproject.toml +pip-compile --no-header --extra dev --extra test --extra nlp -o all_requirements.txt ./../pyproject.toml diff --git a/requirements/dev_requirements.txt b/requirements/dev_requirements.txt index dd9d3685..ba2d141c 100644 --- a/requirements/dev_requirements.txt +++ b/requirements/dev_requirements.txt @@ -6,10 +6,6 @@ aiosignal==1.3.1 # via aiohttp alabaster==0.7.12 # via sphinx -appnope==0.1.3 - # via - # ipykernel - # ipython asttokens==2.2.1 # via stack-data async-timeout==4.0.2 @@ -55,6 +51,8 @@ click==8.1.3 # pip-tools # sec-certs (./../pyproject.toml) # typer +cmake==3.27.7 + # via triton comm==0.1.2 # via ipykernel confection==0.0.3 @@ -62,7 +60,9 @@ confection==0.0.3 contourpy==1.0.6 # via matplotlib coverage[toml]==6.5.0 - # via pytest-cov + # via + # coverage + # pytest-cov cycler==0.11.0 # via matplotlib cymem==2.0.7 @@ -108,6 +108,7 @@ filelock==3.8.2 # huggingface-hub # torch # transformers + # triton # virtualenv fonttools==4.38.0 # via matplotlib @@ -119,6 +120,7 @@ fsspec[http]==2023.1.0 # via # datasets # evaluate + # fsspec gprof2dot==2022.7.29 # via pytest-profiling greenlet==2.0.1 @@ -192,6 +194,8 @@ langcodes==3.3.0 # via spacy langdetect==1.0.9 # via sec-certs (./../pyproject.toml) +lit==17.0.3 + # via triton lxml==4.9.1 # via # pikepdf @@ -284,6 +288,31 @@ numpy==1.23.5 # thinc # torchvision # transformers +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.7.101 + # via torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +nvidia-cufft-cu11==10.9.0.58 + # via torch +nvidia-curand-cu11==10.2.10.91 + # via torch +nvidia-cusolver-cu11==11.4.0.1 + # via torch +nvidia-cusparse-cu11==11.7.4.91 + # via torch +nvidia-nccl-cu11==2.14.3 + # via torch +nvidia-nvtx-cu11==11.7.91 + # via torch packaging==22.0 # via # black @@ -378,7 +407,9 @@ pygments==2.13.0 pyparsing==3.0.9 # via matplotlib pypdf[crypto]==3.2.1 - # via sec-certs (./../pyproject.toml) + # via + # pypdf + # sec-certs (./../pyproject.toml) pyrsistent==0.19.2 # via jsonschema pysankeybeta==1.4.0 @@ -541,6 +572,7 @@ torch==2.0.1 # via # sentence-transformers # torchvision + # triton torchvision==0.15.2 # via sentence-transformers tornado==6.3.2 @@ -570,6 +602,8 @@ traitlets==5.6.0 # nbformat transformers==4.27.1 # via sentence-transformers +triton==2.0.0 + # via torch typer==0.7.0 # via # pathy @@ -607,6 +641,12 @@ webencodings==0.5.1 # via html5lib wheel==0.38.4 # via + # nvidia-cublas-cu11 + # nvidia-cuda-cupti-cu11 + # nvidia-cuda-runtime-cu11 + # nvidia-curand-cu11 + # nvidia-cusparse-cu11 + # nvidia-nvtx-cu11 # pip-tools # pytest-monitor widgetsnbextension==4.0.4 diff --git a/requirements/nlp_requirements.txt b/requirements/nlp_requirements.txt new file mode 100644 index 00000000..dd5366b6 --- /dev/null +++ b/requirements/nlp_requirements.txt @@ -0,0 +1,557 @@ +aiohttp==3.8.6 + # via + # datasets + # fsspec +aiosignal==1.3.1 + # via aiohttp +alembic==1.12.1 + # via optuna +annotated-types==0.6.0 + # via pydantic +asttokens==2.4.1 + # via stack-data +async-timeout==4.0.3 + # via aiohttp +attrs==23.1.0 + # via + # aiohttp + # jsonschema + # referencing +beautifulsoup4==4.12.2 + # via sec-certs (./../pyproject.toml) +blis==0.7.11 + # via thinc +catalogue==2.0.10 + # via + # spacy + # srsly + # thinc +catboost==1.2.2 + # via sec-certs (./../pyproject.toml) +certifi==2023.7.22 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via + # aiohttp + # requests +click==8.1.7 + # via + # nltk + # sec-certs (./../pyproject.toml) + # typer +cloudpathlib==0.16.0 + # via weasel +colorlog==6.7.0 + # via optuna +comm==0.2.0 + # via + # ipykernel + # ipywidgets +confection==0.1.3 + # via + # thinc + # weasel +contourpy==1.2.0 + # via matplotlib +cryptography==41.0.5 + # via pypdf +cycler==0.12.1 + # via matplotlib +cymem==2.0.8 + # via + # preshed + # spacy + # thinc +datasets==2.14.6 + # via + # evaluate + # setfit +debugpy==1.8.0 + # via ipykernel +decorator==5.1.1 + # via ipython +deprecated==1.2.14 + # via pikepdf +dill==0.3.7 + # via + # datasets + # evaluate + # multiprocess +distro==1.8.0 + # via tabula-py +evaluate==0.4.1 + # via setfit +executing==2.0.1 + # via stack-data +filelock==3.13.1 + # via + # huggingface-hub + # torch + # transformers + # triton +fonttools==4.44.0 + # via matplotlib +frozenlist==1.4.0 + # via + # aiohttp + # aiosignal +fsspec[http]==2023.10.0 + # via + # datasets + # evaluate + # fsspec + # huggingface-hub + # torch +graphviz==0.20.1 + # via catboost +greenlet==3.0.1 + # via sqlalchemy +html5lib==1.1 + # via sec-certs (./../pyproject.toml) +huggingface-hub==0.17.3 + # via + # datasets + # evaluate + # sentence-transformers + # tokenizers + # transformers +idna==3.4 + # via + # requests + # yarl +ipykernel==6.26.0 + # via sec-certs (./../pyproject.toml) +ipython==8.17.2 + # via + # ipykernel + # ipywidgets +ipywidgets==8.1.1 + # via sec-certs (./../pyproject.toml) +jedi==0.19.1 + # via ipython +jinja2==3.1.2 + # via + # spacy + # torch +joblib==1.3.2 + # via + # nltk + # pynndescent + # scikit-learn +jpype1==1.4.1 + # via tabula-py +jsonschema==4.19.2 + # via sec-certs (./../pyproject.toml) +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==8.6.0 + # via ipykernel +jupyter-core==5.5.0 + # via + # ipykernel + # jupyter-client +jupyterlab-widgets==3.0.9 + # via ipywidgets +kiwisolver==1.4.5 + # via matplotlib +langcodes==3.3.0 + # via spacy +langdetect==1.0.9 + # via sec-certs (./../pyproject.toml) +llvmlite==0.41.1 + # via + # numba + # pynndescent +lxml==4.9.3 + # via + # pikepdf + # sec-certs (./../pyproject.toml) +mako==1.3.0 + # via alembic +markupsafe==2.1.3 + # via + # jinja2 + # mako +matplotlib==3.8.1 + # via + # catboost + # pysankeybeta + # seaborn + # sec-certs (./../pyproject.toml) +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +mpmath==1.3.0 + # via sympy +multidict==6.0.4 + # via + # aiohttp + # yarl +multiprocess==0.70.15 + # via + # datasets + # evaluate +murmurhash==1.0.10 + # via + # preshed + # spacy + # thinc +nest-asyncio==1.5.8 + # via ipykernel +networkx==3.2.1 + # via + # sec-certs (./../pyproject.toml) + # torch +nltk==3.8.1 + # via sentence-transformers +numba==0.58.1 + # via + # pynndescent + # umap-learn +numpy==1.26.2 + # via + # blis + # catboost + # contourpy + # datasets + # evaluate + # matplotlib + # numba + # optuna + # pandas + # pyarrow + # pysankeybeta + # scikit-learn + # scipy + # seaborn + # sec-certs (./../pyproject.toml) + # sentence-transformers + # spacy + # tabula-py + # thinc + # torchvision + # transformers + # umap-learn +nvidia-cublas-cu12==12.1.3.1 + # via + # nvidia-cudnn-cu12 + # nvidia-cusolver-cu12 + # torch +nvidia-cuda-cupti-cu12==12.1.105 + # via torch +nvidia-cuda-nvrtc-cu12==12.1.105 + # via torch +nvidia-cuda-runtime-cu12==12.1.105 + # via torch +nvidia-cudnn-cu12==8.9.2.26 + # via torch +nvidia-cufft-cu12==11.0.2.54 + # via torch +nvidia-curand-cu12==10.3.2.106 + # via torch +nvidia-cusolver-cu12==11.4.5.107 + # via torch +nvidia-cusparse-cu12==12.1.0.106 + # via + # nvidia-cusolver-cu12 + # torch +nvidia-nccl-cu12==2.18.1 + # via torch +nvidia-nvjitlink-cu12==12.3.52 + # via + # nvidia-cusolver-cu12 + # nvidia-cusparse-cu12 +nvidia-nvtx-cu12==12.1.105 + # via torch +optuna==3.4.0 + # via sec-certs (./../pyproject.toml) +packaging==23.2 + # via + # datasets + # evaluate + # huggingface-hub + # ipykernel + # jpype1 + # matplotlib + # optuna + # pikepdf + # plotly + # setuptools-scm + # spacy + # thinc + # transformers + # weasel +pandas==2.1.3 + # via + # catboost + # datasets + # evaluate + # pysankeybeta + # seaborn + # sec-certs (./../pyproject.toml) + # tabula-py +parso==0.8.3 + # via jedi +pdftotext==2.2.2 + # via sec-certs (./../pyproject.toml) +pexpect==4.8.0 + # via ipython +pikepdf==8.7.1 + # via sec-certs (./../pyproject.toml) +pillow==10.1.0 + # via + # matplotlib + # pikepdf + # sec-certs (./../pyproject.toml) + # torchvision +pkgconfig==1.5.5 + # via sec-certs (./../pyproject.toml) +platformdirs==4.0.0 + # via jupyter-core +plotly==5.18.0 + # via + # catboost + # sec-certs (./../pyproject.toml) +preshed==3.0.9 + # via + # spacy + # thinc +prompt-toolkit==3.0.40 + # via ipython +psutil==5.9.6 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pyarrow==14.0.1 + # via datasets +pycparser==2.21 + # via cffi +pydantic==2.5.0 + # via + # confection + # sec-certs (./../pyproject.toml) + # spacy + # thinc + # weasel +pydantic-core==2.14.1 + # via pydantic +pygments==2.16.1 + # via ipython +pynndescent==0.5.10 + # via umap-learn +pyparsing==3.1.1 + # via matplotlib +pypdf[crypto]==3.17.1 + # via + # pypdf + # sec-certs (./../pyproject.toml) +pysankeybeta==1.4.1 + # via sec-certs (./../pyproject.toml) +python-dateutil==2.8.2 + # via + # jupyter-client + # matplotlib + # pandas + # sec-certs (./../pyproject.toml) +pytz==2023.3.post1 + # via pandas +pyyaml==6.0.1 + # via + # datasets + # huggingface-hub + # optuna + # sec-certs (./../pyproject.toml) + # transformers +pyzmq==25.1.1 + # via + # ipykernel + # jupyter-client +rapidfuzz==3.5.2 + # via sec-certs (./../pyproject.toml) +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications +regex==2023.10.3 + # via + # nltk + # transformers +requests==2.31.0 + # via + # datasets + # evaluate + # fsspec + # huggingface-hub + # responses + # sec-certs (./../pyproject.toml) + # spacy + # torchvision + # transformers + # weasel +responses==0.18.0 + # via evaluate +rpds-py==0.12.0 + # via + # jsonschema + # referencing +safetensors==0.4.0 + # via transformers +scikit-learn==1.3.2 + # via + # pynndescent + # sec-certs (./../pyproject.toml) + # sentence-transformers + # umap-learn +scipy==1.11.3 + # via + # catboost + # pynndescent + # scikit-learn + # sec-certs (./../pyproject.toml) + # sentence-transformers + # umap-learn +seaborn==0.13.0 + # via + # pysankeybeta + # sec-certs (./../pyproject.toml) +sentence-transformers==2.2.2 + # via setfit +sentencepiece==0.1.99 + # via sentence-transformers +setfit==0.7.0 + # via sec-certs (./../pyproject.toml) +setuptools-scm==8.0.4 + # via sec-certs (./../pyproject.toml) +six==1.16.0 + # via + # asttokens + # catboost + # html5lib + # langdetect + # python-dateutil +sklearn==0.0.post11 + # via sec-certs (./../pyproject.toml) +smart-open==6.4.0 + # via + # spacy + # weasel +soupsieve==2.5 + # via beautifulsoup4 +spacy==3.7.2 + # via sec-certs (./../pyproject.toml) +spacy-legacy==3.0.12 + # via spacy +spacy-loggers==1.0.5 + # via spacy +sqlalchemy==2.0.23 + # via + # alembic + # optuna +srsly==2.4.8 + # via + # confection + # spacy + # thinc + # weasel +stack-data==0.6.3 + # via ipython +sympy==1.12 + # via torch +tabula-py==2.8.2 + # via sec-certs (./../pyproject.toml) +tbb==2021.10.0 + # via umap-learn +tenacity==8.2.3 + # via plotly +thinc==8.2.1 + # via spacy +threadpoolctl==3.2.0 + # via scikit-learn +tokenizers==0.14.1 + # via transformers +torch==2.1.0 + # via + # sentence-transformers + # torchvision +torchvision==0.16.0 + # via sentence-transformers +tornado==6.3.3 + # via + # ipykernel + # jupyter-client +tqdm==4.66.1 + # via + # datasets + # evaluate + # huggingface-hub + # nltk + # optuna + # sec-certs (./../pyproject.toml) + # sentence-transformers + # spacy + # transformers + # umap-learn +traitlets==5.13.0 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline +transformers==4.35.0 + # via sentence-transformers +triton==2.1.0 + # via torch +typer==0.9.0 + # via + # spacy + # weasel +typing-extensions==4.8.0 + # via + # alembic + # huggingface-hub + # pydantic + # pydantic-core + # setuptools-scm + # sqlalchemy + # torch + # typer +tzdata==2023.3 + # via pandas +umap-learn==0.5.4 + # via sec-certs (./../pyproject.toml) +urllib3==2.1.0 + # via + # requests + # responses +wasabi==1.1.2 + # via + # spacy + # thinc + # weasel +wcwidth==0.2.10 + # via prompt-toolkit +weasel==0.3.4 + # via spacy +webencodings==0.5.1 + # via html5lib +widgetsnbextension==4.0.9 + # via ipywidgets +wrapt==1.16.0 + # via deprecated +xxhash==3.4.1 + # via + # datasets + # evaluate +yarl==1.9.2 + # via aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 03a85889..17400016 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,10 +4,6 @@ aiohttp==3.8.4 # fsspec aiosignal==1.3.1 # via aiohttp -appnope==0.1.3 - # via - # ipykernel - # ipython asttokens==2.2.1 # via stack-data async-timeout==4.0.2 @@ -38,6 +34,8 @@ click==8.1.3 # nltk # sec-certs (./../pyproject.toml) # typer +cmake==3.27.7 + # via triton comm==0.1.2 # via ipykernel confection==0.0.3 @@ -79,6 +77,7 @@ filelock==3.12.0 # huggingface-hub # torch # transformers + # triton fonttools==4.38.0 # via matplotlib frozenlist==1.3.3 @@ -89,6 +88,7 @@ fsspec[http]==2023.5.0 # via # datasets # evaluate + # fsspec # huggingface-hub html5lib==1.1 # via sec-certs (./../pyproject.toml) @@ -136,6 +136,8 @@ langcodes==3.3.0 # via spacy langdetect==1.0.9 # via sec-certs (./../pyproject.toml) +lit==17.0.3 + # via triton lxml==4.9.1 # via # pikepdf @@ -196,6 +198,31 @@ numpy==1.23.5 # thinc # torchvision # transformers +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.7.101 + # via torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +nvidia-cufft-cu11==10.9.0.58 + # via torch +nvidia-curand-cu11==10.2.10.91 + # via torch +nvidia-cusolver-cu11==11.4.0.1 + # via torch +nvidia-cusparse-cu11==11.7.4.91 + # via torch +nvidia-nccl-cu11==2.14.3 + # via torch +nvidia-nvtx-cu11==11.7.91 + # via torch packaging==22.0 # via # datasets @@ -265,7 +292,9 @@ pygments==2.13.0 pyparsing==3.0.9 # via matplotlib pypdf[crypto]==3.2.1 - # via sec-certs (./../pyproject.toml) + # via + # pypdf + # sec-certs (./../pyproject.toml) pyrsistent==0.19.2 # via jsonschema pysankeybeta==1.4.0 @@ -369,6 +398,7 @@ torch==2.0.1 # via # sentence-transformers # torchvision + # triton torchvision==0.15.2 # via sentence-transformers tornado==6.3.2 @@ -396,6 +426,8 @@ traitlets==5.6.0 # matplotlib-inline transformers==4.29.2 # via sentence-transformers +triton==2.0.0 + # via torch typer==0.7.0 # via # pathy @@ -418,6 +450,14 @@ wcwidth==0.2.5 # via prompt-toolkit webencodings==0.5.1 # via html5lib +wheel==0.41.2 + # via + # nvidia-cublas-cu11 + # nvidia-cuda-cupti-cu11 + # nvidia-cuda-runtime-cu11 + # nvidia-curand-cu11 + # nvidia-cusparse-cu11 + # nvidia-nvtx-cu11 widgetsnbextension==4.0.4 # via ipywidgets xxhash==3.2.0 diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index 0981a4ea..e4c8c8f7 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -4,10 +4,6 @@ aiohttp==3.8.4 # fsspec aiosignal==1.3.1 # via aiohttp -appnope==0.1.3 - # via - # ipykernel - # ipython asttokens==2.2.1 # via stack-data async-timeout==4.0.2 @@ -39,6 +35,8 @@ click==8.1.3 # nltk # sec-certs (./../pyproject.toml) # typer +cmake==3.27.7 + # via triton comm==0.1.2 # via ipykernel confection==0.0.3 @@ -84,6 +82,7 @@ filelock==3.12.0 # huggingface-hub # torch # transformers + # triton fonttools==4.38.0 # via matplotlib frozenlist==1.3.3 @@ -94,6 +93,7 @@ fsspec[http]==2023.5.0 # via # datasets # evaluate + # fsspec # huggingface-hub html5lib==1.1 # via sec-certs (./../pyproject.toml) @@ -143,6 +143,8 @@ langcodes==3.3.0 # via spacy langdetect==1.0.9 # via sec-certs (./../pyproject.toml) +lit==17.0.3 + # via triton lxml==4.9.1 # via # pikepdf @@ -203,6 +205,31 @@ numpy==1.23.5 # thinc # torchvision # transformers +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # nvidia-cusolver-cu11 + # torch +nvidia-cuda-cupti-cu11==11.7.101 + # via torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +nvidia-cufft-cu11==10.9.0.58 + # via torch +nvidia-curand-cu11==10.2.10.91 + # via torch +nvidia-cusolver-cu11==11.4.0.1 + # via torch +nvidia-cusparse-cu11==11.7.4.91 + # via torch +nvidia-nccl-cu11==2.14.3 + # via torch +nvidia-nvtx-cu11==11.7.91 + # via torch packaging==22.0 # via # datasets @@ -275,7 +302,9 @@ pygments==2.13.0 pyparsing==3.0.9 # via matplotlib pypdf[crypto]==3.2.1 - # via sec-certs (./../pyproject.toml) + # via + # pypdf + # sec-certs (./../pyproject.toml) pyrsistent==0.19.2 # via jsonschema pysankeybeta==1.4.0 @@ -385,6 +414,7 @@ torch==2.0.1 # via # sentence-transformers # torchvision + # triton torchvision==0.15.2 # via sentence-transformers tornado==6.3.2 @@ -412,6 +442,8 @@ traitlets==5.6.0 # matplotlib-inline transformers==4.29.2 # via sentence-transformers +triton==2.0.0 + # via torch typer==0.7.0 # via # pathy @@ -434,6 +466,14 @@ wcwidth==0.2.5 # via prompt-toolkit webencodings==0.5.1 # via html5lib +wheel==0.41.2 + # via + # nvidia-cublas-cu11 + # nvidia-cuda-cupti-cu11 + # nvidia-cuda-runtime-cu11 + # nvidia-curand-cu11 + # nvidia-cusparse-cu11 + # nvidia-nvtx-cu11 widgetsnbextension==4.0.4 # via ipywidgets xxhash==3.2.0 diff --git a/src/sec_certs/constants.py b/src/sec_certs/constants.py index b28b925d..773a5720 100644 --- a/src/sec_certs/constants.py +++ b/src/sec_certs/constants.py @@ -1,6 +1,11 @@ import re from pathlib import Path -from typing import Final +from typing import Final, Literal + +RANDOM_STATE: Final[int] = 42 +REF_ANNOTATION_MODES = Literal["training", "evaluation", "production", "cross-validation"] +REF_EMBEDDING_METHOD = Literal["tf_idf", "transformer"] + DUMMY_NONEXISTING_PATH = Path("/this/is/dummy/nonexisting/path") diff --git a/src/sec_certs/data/reference_annotations/readme.md b/src/sec_certs/data/reference_annotations/readme.md index b10a2e30..8521eead 100644 --- a/src/sec_certs/data/reference_annotations/readme.md +++ b/src/sec_certs/data/reference_annotations/readme.md @@ -54,35 +54,8 @@ These can be further merged into the following super-categories: The inter-annotator agreement is measured both with Cohen's Kappa and with percentage. The results are as follows: | Cohen's Kappa | Percentage | -|---------------|------------| +| ------------- | ---------- | | 0.71 | 0.82 | -The code used to measure the agreement is: +The code used to measure the agreement is stored in `notebooks/cc/reference_annotations/inter_annotator_agreement.ipynb`. -```python -import pandas as pd -from pathlib import Path -from sklearn.metrics import cohen_kappa_score - -def load_all_dataframes(base_folder: Path) -> pd.DataFrame: - splits = ["train", "valid", "test"] - - df_train, df_valid, df_test = pd.DataFrame(), pd.DataFrame(), pd.DataFrame() - for split in splits: - df = pd.read_csv(base_folder / f"{split}.csv") - if split == "train": - df_train = df - elif split == "valid": - df_valid = df - else: - df_test = df - - return pd.concat([df_train, df_valid, df_test]) - -adam_df = load_all_dataframes(Path("./src/sec_certs/data/reference_annotations/adam")) -jano_df = load_all_dataframes(Path("./src/sec_certs/data/reference_annotations/jano")) -agreement_series = adam_df.label == jano_df.label - -print(f"Cohen's Kappa: {cohen_kappa_score(adam_df.label, jano_df.label)}") -print(f"Percentage agreement: {agreement_series.loc[agreement_series == True].count() / agreement_series.count()}") -``` diff --git a/src/sec_certs/dataset/cc.py b/src/sec_certs/dataset/cc.py index ddd633e7..afde8bd2 100644 --- a/src/sec_certs/dataset/cc.py +++ b/src/sec_certs/dataset/cc.py @@ -7,7 +7,7 @@ import tempfile from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import ClassVar, Iterator, Literal, cast +from typing import ClassVar, Iterator, cast import numpy as np import pandas as pd @@ -22,14 +22,11 @@ from sec_certs.dataset.cve import CVEDataset from sec_certs.dataset.dataset import AuxiliaryDatasets, Dataset, logger from sec_certs.dataset.protection_profile import ProtectionProfileDataset from sec_certs.model import ( - ReferenceAnnotator, - ReferenceAnnotatorTrainer, ReferenceFinder, SARTransformer, TransitiveVulnerabilityFinder, ) from sec_certs.model.cc_matching import CCSchemeMatcher -from sec_certs.model.references.segment_extractor import ReferenceSegmentExtractor from sec_certs.sample.cc import CCCertificate from sec_certs.sample.cc_certificate_id import CertificateId from sec_certs.sample.cc_maintenance_update import CCMaintenanceUpdate @@ -38,7 +35,6 @@ from sec_certs.sample.protection_profile import ProtectionProfile from sec_certs.serialization.json import ComplexSerializableType, serialize from sec_certs.utils import helpers from sec_certs.utils import parallel_processing as cert_processing -from sec_certs.utils.nlp import prec_recall_metric @dataclass @@ -729,7 +725,6 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable self._compute_scheme_data() self._compute_cert_labs() self._compute_sars() - self.annotate_references() def _compute_sars(self) -> None: logger.info("Computing heuristics: Computing SARs") @@ -838,68 +833,6 @@ class CCDataset(Dataset[CCCertificate, CCAuxiliaryDatasets], ComplexSerializable return update_dset - def annotate_references(self, mode: Literal["training", "production"] = "production"): - """ - Fills in `cert.heuristics.annotated_references` with reference labels. - This requires (a) a pre-trained `ReferenceAnnotator` or (b) to train a `ReferenceAnnotator`. - The behaviour is controlled by config keys `config.cc_reference_annotator_should_train` and - `config.cc_reference_annotator_path`. If should_train is False and path is provided, a pre-trained annotator - will be loaded and used. - """ - if not self.state.pdfs_converted: - logger.info( - "Attempting run analysis of txt files while not having the pdf->txt conversion done. Returning." - ) - return - if not self.state.auxiliary_datasets_processed: - logger.info( - "Attempting to run analysis of certifies while not having the auxiliary datasets processed. Returning." - ) - - if not config.cc_reference_annotator_should_train: - model_dir = ( - config.cc_reference_annotator_dir if config.cc_reference_annotator_dir else self.reference_annotator_dir - ) - try: - annotator = ReferenceAnnotator.from_pretrained(model_dir) - except Exception: - logger.error( - "annotate_references() method was called with `config.cc_reference_annotator_should_train=False`." - f"Further, the model was not found either at `config.cc_reference_annotator_dir={config.cc_reference_annotator_dir}`" - f"nor at {self.reference_annotator_dir}. Either: (a) allow training with `config.cc_reference_annotator_should_train=True`;" - "(b) set path to model with `config.cc_reference_annotator_path`; (c) paste the model into {self.reference_annotator_dir}. Returning." - ) - return - - logger.info("Extracting segments of text relevant for reference annotations.") - df = ReferenceSegmentExtractor()(self.certs.values()) - if config.cc_reference_annotator_should_train: - annotator = self._train_reference_annotator(df, mode=mode) - - logger.info("Predicting reference labels") - df = annotator.predict_df(df) - refs: dict[str, dict[str, str]] = dict.fromkeys(df.dgst, {}) - for dgst, cert_id, label in zip(df.dgst, df.referenced_cert_id, df.y_pred): - refs[dgst][cert_id] = label - - for dgst, value in refs.items(): - self[dgst].heuristics.annotated_references = value - - def _train_reference_annotator( - self, df: pd.DataFrame, save_model: bool = True, mode: Literal["training", "production"] = "production" - ) -> ReferenceAnnotator: - trainer = ReferenceAnnotatorTrainer.from_df(df, prec_recall_metric, "transformer", mode) - logger.info( - "Training ReferenceAnnotator on {df.shape[0]} samples ({df.loc[df.split == 'train'].shape[0]}/{df.loc[df.split == 'valid'].shape[0]}/{df.loc[df.split == 'test'].shape[0]}) (train/valid/test)." - ) - trainer.train() - logger.info(trainer.evaluate()) - - if save_model: - trainer.clf.save_pretrained(self.reference_annotator_dir) - - return trainer.clf - def process_schemes(self, to_download: bool = True, only_schemes: set[str] | None = None) -> CCSchemeDataset: """ Downloads or loads from json a dataset of CC scheme data. diff --git a/src/sec_certs/dataset/fips.py b/src/sec_certs/dataset/fips.py index f5dda10d..24536b9b 100644 --- a/src/sec_certs/dataset/fips.py +++ b/src/sec_certs/dataset/fips.py @@ -17,7 +17,7 @@ from sec_certs.dataset.cpe import CPEDataset from sec_certs.dataset.cve import CVEDataset from sec_certs.dataset.dataset import AuxiliaryDatasets, Dataset from sec_certs.dataset.fips_algorithm import FIPSAlgorithmDataset -from sec_certs.model.references.reference_finder import ReferenceFinder +from sec_certs.model.reference_finder import ReferenceFinder from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder from sec_certs.sample.fips import FIPSCertificate from sec_certs.serialization.json import ComplexSerializableType, serialize diff --git a/src/sec_certs/model/__init__.py b/src/sec_certs/model/__init__.py index 4a3a14c0..69cbe194 100644 --- a/src/sec_certs/model/__init__.py +++ b/src/sec_certs/model/__init__.py @@ -6,10 +6,7 @@ leveraged by members of Dataset package and are directly applied on members of S from sec_certs.model.cc_matching import CCSchemeMatcher from sec_certs.model.cpe_matching import CPEClassifier from sec_certs.model.fips_matching import FIPSProcessMatcher -from sec_certs.model.references.annotator import ReferenceAnnotator -from sec_certs.model.references.annotator_trainer import ReferenceAnnotatorTrainer -from sec_certs.model.references.reference_finder import ReferenceFinder -from sec_certs.model.references.segment_extractor import ReferenceSegmentExtractor +from sec_certs.model.reference_finder import ReferenceFinder from sec_certs.model.sar_transformer import SARTransformer from sec_certs.model.transitive_vulnerability_finder import TransitiveVulnerabilityFinder diff --git a/src/sec_certs/model/reference_finder.py b/src/sec_certs/model/reference_finder.py new file mode 100644 index 00000000..94a3b29f --- /dev/null +++ b/src/sec_certs/model/reference_finder.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from typing import Callable, Dict, List, Optional, Set, TypeVar + +from sec_certs.sample.certificate import Certificate, References + +CertSubType = TypeVar("CertSubType", bound=Certificate) +Certificates = Dict[str, CertSubType] +ReferencedByDirect = Dict[str, Set[str]] +ReferencedByIndirect = Dict[str, Set[str]] +ReferencesType = Dict[str, Dict[str, Optional[Set[str]]]] +IDMapping = Dict[str, List[str]] +UnknownReferences = Dict[str, Set[str]] +IDLookupFunc = Callable[[CertSubType], str] +ReferenceLookupFunc = Callable[[CertSubType], Set[str]] + + +# TODO: All of this can and should be rewritten on top of networkx or some other graph library. +class ReferenceFinder: + """ + The class assigns references of other certificate instances for each instance. + Adheres to sklearn BaseEstimator interface. + The fit is called on a dictionary of certificates, builds a hashmap of references, and assigns references for each certificate in the dictionary. + """ + + def __init__(self: ReferenceFinder) -> None: + self.references: ReferencesType = {} + self.id_mapping: IDMapping = {} + self._fitted: bool = False + + def _create_id_mapping(self, certificates: Certificates, id_func: IDLookupFunc) -> None: + """ + Create the ID mapping of certificate IDs to certificate digests. + + Necessary for handling duplicates. + """ + # Create a mapping of certificate ID to certificate digests with that ID. + for dgst in certificates: + cert_id = id_func(certificates[dgst]) + c_list = self.id_mapping.setdefault(cert_id, []) + c_list.append(dgst) + + # Sort digests in ID mapping to have deterministic behavior. + # The certificate with the first digest will be used with that ID, others will be discarded. + for digests in self.id_mapping.values(): + digests.sort() + + def _compute_indirect_references(self, referenced_by: ReferencedByDirect) -> ReferencedByIndirect: + """ + Compute indirect references via a BFS algorithm. + """ + referenced_by_indirect: ReferencedByIndirect = {} + + # Populate with direct references. + certs_id_list = referenced_by.keys() + for cert_id in certs_id_list: + referenced_by_indirect[cert_id] = set() + for item in referenced_by[cert_id]: + referenced_by_indirect[cert_id].add(item) + + # Flood in the indirect ones. + new_change_detected = True + while new_change_detected: + new_change_detected = False + + for cert_id in certs_id_list: + tmp_referenced_by_indirect_nums = referenced_by_indirect[cert_id].copy() + for referencing in tmp_referenced_by_indirect_nums: + if referencing in certs_id_list: + tmp_referencing = referenced_by_indirect[referencing].copy() + newly_discovered_references = [ + x for x in tmp_referencing if x not in referenced_by_indirect[cert_id] + ] + referenced_by_indirect[cert_id].update(newly_discovered_references) + if newly_discovered_references: + new_change_detected = True + return referenced_by_indirect + + def _build_referenced_by( + self, certificates: Certificates, ref_lookup_func: ReferenceLookupFunc + ) -> tuple[ReferencedByDirect, ReferencedByIndirect]: + referenced_by: ReferencedByDirect = {} + + for this_cert_id, cert_digests in self.id_mapping.items(): + # Take the first certificate digest from the ID mapping (to ensure deterministic behavior and resolve duplicates). + # TODO: A better approach for handling duplicates in the future would be nice. + cert_dgst = cert_digests[0] + cert_obj = certificates[cert_dgst] + + refs = ref_lookup_func(cert_obj) + if refs is None: + continue + + # Process direct reference + # All are added here, the unknown ones are filtered later on. + for cert_id in refs: + if cert_id == this_cert_id: + continue + referenced_by.setdefault(cert_id, set()) + referenced_by[cert_id].add(this_cert_id) + + # Now do the indirect ones + referenced_by_indirect = self._compute_indirect_references(referenced_by) + return referenced_by, referenced_by_indirect + + def _get_reverse_references( + self, cert_id: str, references: ReferencedByDirect | ReferencedByIndirect + ) -> set[str] | None: + result = set() + + for other_id in references: + if cert_id in references[other_id]: + result.add(other_id) + + return result if result else None + + def _build_referencing( + self, referenced_by_direct: ReferencedByDirect, referenced_by_indirect: ReferencedByIndirect + ) -> None: + for cert_id, cert_digests in self.id_mapping.items(): + cert_dgst = cert_digests[0] + self.references[cert_dgst] = { + "directly_referenced_by": referenced_by_direct.get(cert_id, None), + "indirectly_referenced_by": referenced_by_indirect.get(cert_id, None), + "directly_referencing": self._get_reverse_references(cert_id, referenced_by_direct), + "indirectly_referencing": self._get_reverse_references(cert_id, referenced_by_indirect), + } + + def fit(self, certificates: Certificates, id_func: IDLookupFunc, ref_lookup_func: ReferenceLookupFunc) -> None: + """ + Builds a list of references and assigns references for each certificate instance. + + :param Certificates certificates: dictionary of certificates with hashes as key + :param IDLookupFunc id_func: lookup function for cert id + :param ReferenceLookupFunc ref_lookup_func: lookup for references + """ + if self._fitted: + raise ValueError("Finder already fitted") + # Create the ID mapping first so that we can resolve duplicates. + self._create_id_mapping(certificates, id_func) + + # Build the referenced_by first + referenced_by_direct, referenced_by_indirect = self._build_referenced_by(certificates, ref_lookup_func) + + # Build the referencing second (this actually writes into self.references). + self._build_referencing(referenced_by_direct, referenced_by_indirect) + self._fitted = True + + @property + def unknown_references(self) -> UnknownReferences: + """ + Get the unknown references in the fitted dataset (to unknown certificate IDs, not in the dataset during fit). + """ + if not self._fitted: + return {} + result = {} + for cert_id, digests in self.id_mapping.items(): + cert_digest = digests[0] + cert_references = self.references[cert_digest] + direct_refs = cert_references["directly_referencing"] + if not direct_refs: + continue + unknowns = set(filter(lambda refd_cert_id: refd_cert_id not in self.id_mapping, direct_refs)) + if unknowns: + result[cert_id] = unknowns + return result + + @property + def duplicates(self) -> IDMapping: + """ + Get the duplicates in the fitted dataset. + + :return IDMapping: Mapping of certificate ID to digests that share it. + """ + if not self._fitted: + return {} + return {cert_id: digests for cert_id, digests in self.id_mapping.items() if len(digests) > 1} + + def predict_single_cert(self, dgst: str, keep_unknowns: bool = True) -> References: + """ + Get the references object for specified certificate digest. + + :param dgst: certificate digest + :param keep_unknowns: Whether to keep references to unknown certificate IDs + :return References: References object + """ + if not self._fitted: + raise ValueError("Finder not yet fitted") + + def wrap(res): + if not res: + return None + # If we do not want the unknown references, filter them here. + if not keep_unknowns: + res = set(filter(lambda cert_id: cert_id in self.id_mapping, res)) + return set(res) if res else None + + if dgst not in self.references: + return References() + + return References( + wrap(self.references[dgst].get("directly_referenced_by", None)), + wrap(self.references[dgst].get("indirectly_referenced_by", None)), + wrap(self.references[dgst].get("directly_referencing", None)), + wrap(self.references[dgst].get("indirectly_referencing", None)), + ) + + def predict(self, dgst_list: list[str], keep_unknowns: bool = True) -> dict[str, References]: + """ + Get the references for a list of certificate digests. + + :param dgst_list: List of certificate digests. + :param keep_unknowns: Whether to keep references to and from unknown certificate IDs + :return Dict[str, References]: Dict with certificate hash and References object. + """ + if not self._fitted: + raise ValueError("Finder not yet fitted") + cert_references = {} + + for dgst in dgst_list: + cert_references[dgst] = self.predict_single_cert(dgst, keep_unknowns=keep_unknowns) + + return cert_references diff --git a/src/sec_certs/model/references/annotator.py b/src/sec_certs/model/references/annotator.py deleted file mode 100644 index 76676895..00000000 --- a/src/sec_certs/model/references/annotator.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -import json -import logging -import re -from collections import Counter -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd -from setfit import SetFitModel - -from sec_certs.utils.nlp import softmax - -logger = logging.getLogger(__name__) - - -@dataclass -class ReferenceAnnotator: - """ - Class for annotating references. Its instances are supposed to by trained by `ReferenceAnnotatorTrainer`. - Can be serialized into a directory / load from a directory. - """ - - _model: Any - _label_mapping: dict[int, str] - _soft_voting_power: int = 2 - _use_analytical_rule_name_similarity: bool = True - - # TODO: This does not load hyperparameters, only the model and label mapping - @classmethod - def from_pretrained(cls, model_dir: str | Path) -> ReferenceAnnotator: - """ - Loads classifier from directory, assuming that: - - the SetFitModel was dumped into that directory with model.save_pretrained(model_dir) - - json file label_mapping.json exists in model_dir - - :param str | Path model_dir: path to directory to search for model and label mapping - :return RerefenceClassifier: classifier with SetFitModel and label mapping - """ - logger.info(f"Loading pre-trained reference annotator from: {model_dir}") - model = SetFitModel.from_pretrained(str(model_dir)) - with (Path(model_dir) / "label_mapping.json").open("r") as handle: - label_mapping = json.load(handle) - label_mapping = {int(k): v for k, v in label_mapping.items()} - - return cls(model, label_mapping) - - # TODO: This does not save hyperparameters, only the model and label mapping - def save_pretrained(self, model_dir: str | Path): - """ - Will dump _model and _label_mapping into a directory. - """ - logger.info(f"Saving ReferenceAnnotator to {model_dir}") - model_dir = Path(model_dir) - model_dir.mkdir(exist_ok=True, parents=True) - logger.info - with (model_dir / "label_mapping.json").open("w") as handle: - json.dump(self._label_mapping, handle, indent=4) - self._model._save_pretrained(str(model_dir)) - - def train(self, train_dataset: pd.DataFrame): - raise NotImplementedError("ReferenceAnnotatorTrainer shall be used for training") - - def predict(self, X: list[list[str]]) -> list[str]: - return [self._predict_single(x) for x in X] - - def _predict_single(self, sample: list[str]) -> str: - # return self._predict_single_majority_vote(sample) - return self._label_mapping[int(np.argmax(self._predict_proba_single(sample)))] - - def predict_proba(self, X: list[list[str]]) -> list[list[float]]: - return [self._predict_proba_single(x) for x in X] - - def _predict_proba_single(self, sample: list[str]) -> list[float]: - """ - 1. Get predictions for each segment, convert pytorch tensor to numpy - 2. Square every prediction to reward confidence - 3. Sum probabilities for each label - 4. softmax - """ - return softmax(np.power(self._model.predict_proba(sample, as_numpy=True), self._soft_voting_power).sum(axis=0)) - - def _predict_single_majority_vote(self, sample: list[str]) -> str: - predictions = self._model.predict(sample) - most_common = Counter(predictions).most_common() - if len(most_common) > 1 and most_common[0][1] == most_common[1][1]: - return self._label_mapping[int(np.argmax(self._predict_proba_single(sample)))] - else: - return self._label_mapping[int(most_common[0][0])] - - def predict_df(self, df: pd.DataFrame) -> pd.DataFrame: - """ - WIll read df.segments and populate the dataframe with predictions. - """ - - def matches_reevaluation(segments: list[str]) -> bool: - regex_a = r"This is a re-?\s?certification based on (the\s){1,2}referenced product" - regex_b = r"Re-?\s?Zertifizierung basierend auf (the\s){1,2}referenced product" - return any( - re.search(regex_a, segment, re.IGNORECASE) or re.search(regex_b, segment, re.IGNORECASE) - for segment in segments - ) - - df_new = df.copy() - y_proba = self.predict_proba(df_new.segments) - df_new["y_proba"] = y_proba - df_new["y_pred"] = self.predict(df_new.segments) - - if self._use_analytical_rule_name_similarity: - df_new.loc[ - (df_new.name_similarity == 100) - & (df_new.name_len_diff < 5) - & ((df_new.y_pred != "RE-EVALUATION") & (df_new.y_pred != "PREVIOUS_VERSION")), - ["y_pred"], - ] = "PREVIOUS_VERSION" - - df_new["maches_reevaluation"] = df_new.segments.map(matches_reevaluation) - df_new.loc[df_new.maches_reevaluation, ["y_pred"]] = "RE-EVALUATION" - - df_new["correct"] = df_new.apply( - lambda row: row["y_pred"] == row["label"] if not pd.isnull(row["label"]) else np.NaN, axis=1 - ) - return df_new diff --git a/src/sec_certs/model/references/annotator_trainer.py b/src/sec_certs/model/references/annotator_trainer.py deleted file mode 100644 index fcc08422..00000000 --- a/src/sec_certs/model/references/annotator_trainer.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import annotations - -import logging -from functools import partial -from typing import Callable, Literal - -import pandas as pd -from datasets import ClassLabel, Dataset, Features, NamedSplit, Value -from sentence_transformers.losses import CosineSimilarityLoss -from setfit import SetFitModel, SetFitTrainer -from sklearn.metrics import f1_score - -from sec_certs.model.references.annotator import ReferenceAnnotator -from sec_certs.utils.nlp import prepare_reference_annotations_df - -logger = logging.getLogger(__name__) - - -class ReferenceAnnotatorTrainer: - def __init__( - self, - train_dataset: pd.DataFrame, - eval_dataset: pd.DataFrame, - metric: Callable, - use_analytical_rule_name_similarity: bool = True, - n_iterations: int = 20, - n_epochs: int = 1, - batch_size: int = 16, - segmenter_metric: Literal["accuracy", "f1"] = "accuracy", - ensemble_soft_voting_power: int = 2, - ): - self._train_dataset = train_dataset - self._eval_dataset = eval_dataset - self._metric = metric - self.use_analytical_rule_name_similarity = use_analytical_rule_name_similarity - self.n_iterations = n_iterations - self.n_epochs = n_epochs - self.batch_size = batch_size - self.segmenter_metric = segmenter_metric - self.ensemble_soft_voting_power = ensemble_soft_voting_power - - self._model, self._trainer, self.label_mapping = self._init_trainer() - - self.clf = ReferenceAnnotator( - self._model, self.label_mapping, self.ensemble_soft_voting_power, self.use_analytical_rule_name_similarity - ) - - @classmethod - def from_df( - cls, - df: pd.DataFrame, - metric: Callable, - mode: Literal["training", "evaluation", "production"] = "training", - use_analytical_rule_name_similarity: bool = True, - n_iterations: int = 20, - n_epochs: int = 1, - batch_size: int = 16, - segmenter_metric: Literal["accuracy", "f1"] = "accuracy", - ensemble_soft_voting_power: int = 2, - ): - df = prepare_reference_annotations_df(df) - dataset_generation_method = { - "training": ReferenceAnnotatorTrainer.split_df_for_training, - "evaluation": ReferenceAnnotatorTrainer.split_df_for_evaluation, - "production": ReferenceAnnotatorTrainer.split_df_for_production, - } - - train_dataset, eval_dataset = dataset_generation_method[mode](df) - return cls( - train_dataset, - eval_dataset, - metric, - use_analytical_rule_name_similarity, - n_iterations, - n_epochs, - batch_size, - segmenter_metric, - ensemble_soft_voting_power, - ) - - @staticmethod - def split_df_for_training(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: - return df.loc[df.split == "train"].drop(columns="split"), df.loc[df.split == "valid"].drop(columns="split") - - @staticmethod - def split_df_for_production(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: - logger.info("Splitting dataset for production, model can be trained, but not evaluated.") - return df.drop(columns="split"), df.drop(df.index) - - @staticmethod - def split_df_for_evaluation(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: - df.split = df.split.map({"test": "test", "train": "train", "valid": "train"}) - if df.loc[df.split == "test"].empty: - logger.warning("`test` split for annotator dataset is empty -> model can be trained, but not evaluated.") - return df.loc[df.split == "train"].drop(columns="split"), df.loc[df.split == "test"].drop(columns="split") - - def _init_trainer(self): - model = SetFitModel.from_pretrained("paraphrase-multilingual-mpnet-base-v2") - # model = SetFitModel.from_pretrained("all-mpnet-base-v2") - - train_dataset_relevant_cols = self._train_dataset[["dgst", "canonical_reference_keyword", "segments", "label"]] - eval_dataset_relevant_cols = self._eval_dataset[["dgst", "canonical_reference_keyword", "segments", "label"]] - internal_train_dataset = self._get_hugging_face_datasets_from_df(train_dataset_relevant_cols, "train") - internal_validation_dataset = self._get_hugging_face_datasets_from_df(eval_dataset_relevant_cols, "validation") - - # Align labels alphabetically - labels_alphabetically = sorted(internal_train_dataset.features["label"].names) - label2id = {label: index for index, label in enumerate(labels_alphabetically)} - internal_train_dataset = internal_train_dataset.align_labels_with_mapping(label2id, "label") - internal_validation_dataset = internal_validation_dataset.align_labels_with_mapping(label2id, "label") - - if self.segmenter_metric == "accuracy": - metric_to_use = "accuracy" - else: - metric_to_use = partial(f1_score, average="weighted", zero_division=0) - - trainer = SetFitTrainer( - model=model, - train_dataset=internal_train_dataset, - eval_dataset=internal_validation_dataset, - loss_class=CosineSimilarityLoss, - metric=metric_to_use, - batch_size=self.batch_size, - num_iterations=self.n_iterations, # The number of text pairs to generate for contrastive learning - num_epochs=self.n_epochs, # The number of epochs to use for contrastive learning - column_mapping={ - "segment": "text", - "label": "label", - }, # Map dataset columns to text/label expected by trainer - ) - return model, trainer, {index: label for label, index in label2id.items()} - - @staticmethod - def _get_hugging_face_datasets_from_df(df: pd.DataFrame, split: NamedSplit) -> Dataset: - df_to_use = df.explode("segments").rename(columns={"segments": "segment"}).loc[df.label.notnull()] - features = Features( - { - "dgst": Value("string"), - "canonical_reference_keyword": Value("string"), - "segment": Value("string"), - "label": ClassLabel(names=list(df_to_use.label.unique())), - } - ) - return Dataset.from_pandas(df_to_use, features=features, split=split, preserve_index=False) - - def train(self): - self._trainer.train(show_progress_bar=True) - - def evaluate(self): - print("Internal evaluation (of model working on individual segments)") - print(self._evaluate_raw()) - print("Actual evaluation after ensemble soft voting") - print(self._evaluate_stacked()) - - def _evaluate_raw(self): - if self._eval_dataset.empty: - logger.error("Evaluation dataset is empty, cannot evaluate, returning.") - return - return self._trainer.evaluate() - - def _evaluate_stacked(self): - y_pred = self.clf.predict(self._eval_dataset.segments) - y_true = self._eval_dataset.label - return self._metric(y_pred, y_true) diff --git a/src/sec_certs/model/references/reference_finder.py b/src/sec_certs/model/references/reference_finder.py deleted file mode 100644 index 94a3b29f..00000000 --- a/src/sec_certs/model/references/reference_finder.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -from typing import Callable, Dict, List, Optional, Set, TypeVar - -from sec_certs.sample.certificate import Certificate, References - -CertSubType = TypeVar("CertSubType", bound=Certificate) -Certificates = Dict[str, CertSubType] -ReferencedByDirect = Dict[str, Set[str]] -ReferencedByIndirect = Dict[str, Set[str]] -ReferencesType = Dict[str, Dict[str, Optional[Set[str]]]] -IDMapping = Dict[str, List[str]] -UnknownReferences = Dict[str, Set[str]] -IDLookupFunc = Callable[[CertSubType], str] -ReferenceLookupFunc = Callable[[CertSubType], Set[str]] - - -# TODO: All of this can and should be rewritten on top of networkx or some other graph library. -class ReferenceFinder: - """ - The class assigns references of other certificate instances for each instance. - Adheres to sklearn BaseEstimator interface. - The fit is called on a dictionary of certificates, builds a hashmap of references, and assigns references for each certificate in the dictionary. - """ - - def __init__(self: ReferenceFinder) -> None: - self.references: ReferencesType = {} - self.id_mapping: IDMapping = {} - self._fitted: bool = False - - def _create_id_mapping(self, certificates: Certificates, id_func: IDLookupFunc) -> None: - """ - Create the ID mapping of certificate IDs to certificate digests. - - Necessary for handling duplicates. - """ - # Create a mapping of certificate ID to certificate digests with that ID. - for dgst in certificates: - cert_id = id_func(certificates[dgst]) - c_list = self.id_mapping.setdefault(cert_id, []) - c_list.append(dgst) - - # Sort digests in ID mapping to have deterministic behavior. - # The certificate with the first digest will be used with that ID, others will be discarded. - for digests in self.id_mapping.values(): - digests.sort() - - def _compute_indirect_references(self, referenced_by: ReferencedByDirect) -> ReferencedByIndirect: - """ - Compute indirect references via a BFS algorithm. - """ - referenced_by_indirect: ReferencedByIndirect = {} - - # Populate with direct references. - certs_id_list = referenced_by.keys() - for cert_id in certs_id_list: - referenced_by_indirect[cert_id] = set() - for item in referenced_by[cert_id]: - referenced_by_indirect[cert_id].add(item) - - # Flood in the indirect ones. - new_change_detected = True - while new_change_detected: - new_change_detected = False - - for cert_id in certs_id_list: - tmp_referenced_by_indirect_nums = referenced_by_indirect[cert_id].copy() - for referencing in tmp_referenced_by_indirect_nums: - if referencing in certs_id_list: - tmp_referencing = referenced_by_indirect[referencing].copy() - newly_discovered_references = [ - x for x in tmp_referencing if x not in referenced_by_indirect[cert_id] - ] - referenced_by_indirect[cert_id].update(newly_discovered_references) - if newly_discovered_references: - new_change_detected = True - return referenced_by_indirect - - def _build_referenced_by( - self, certificates: Certificates, ref_lookup_func: ReferenceLookupFunc - ) -> tuple[ReferencedByDirect, ReferencedByIndirect]: - referenced_by: ReferencedByDirect = {} - - for this_cert_id, cert_digests in self.id_mapping.items(): - # Take the first certificate digest from the ID mapping (to ensure deterministic behavior and resolve duplicates). - # TODO: A better approach for handling duplicates in the future would be nice. - cert_dgst = cert_digests[0] - cert_obj = certificates[cert_dgst] - - refs = ref_lookup_func(cert_obj) - if refs is None: - continue - - # Process direct reference - # All are added here, the unknown ones are filtered later on. - for cert_id in refs: - if cert_id == this_cert_id: - continue - referenced_by.setdefault(cert_id, set()) - referenced_by[cert_id].add(this_cert_id) - - # Now do the indirect ones - referenced_by_indirect = self._compute_indirect_references(referenced_by) - return referenced_by, referenced_by_indirect - - def _get_reverse_references( - self, cert_id: str, references: ReferencedByDirect | ReferencedByIndirect - ) -> set[str] | None: - result = set() - - for other_id in references: - if cert_id in references[other_id]: - result.add(other_id) - - return result if result else None - - def _build_referencing( - self, referenced_by_direct: ReferencedByDirect, referenced_by_indirect: ReferencedByIndirect - ) -> None: - for cert_id, cert_digests in self.id_mapping.items(): - cert_dgst = cert_digests[0] - self.references[cert_dgst] = { - "directly_referenced_by": referenced_by_direct.get(cert_id, None), - "indirectly_referenced_by": referenced_by_indirect.get(cert_id, None), - "directly_referencing": self._get_reverse_references(cert_id, referenced_by_direct), - "indirectly_referencing": self._get_reverse_references(cert_id, referenced_by_indirect), - } - - def fit(self, certificates: Certificates, id_func: IDLookupFunc, ref_lookup_func: ReferenceLookupFunc) -> None: - """ - Builds a list of references and assigns references for each certificate instance. - - :param Certificates certificates: dictionary of certificates with hashes as key - :param IDLookupFunc id_func: lookup function for cert id - :param ReferenceLookupFunc ref_lookup_func: lookup for references - """ - if self._fitted: - raise ValueError("Finder already fitted") - # Create the ID mapping first so that we can resolve duplicates. - self._create_id_mapping(certificates, id_func) - - # Build the referenced_by first - referenced_by_direct, referenced_by_indirect = self._build_referenced_by(certificates, ref_lookup_func) - - # Build the referencing second (this actually writes into self.references). - self._build_referencing(referenced_by_direct, referenced_by_indirect) - self._fitted = True - - @property - def unknown_references(self) -> UnknownReferences: - """ - Get the unknown references in the fitted dataset (to unknown certificate IDs, not in the dataset during fit). - """ - if not self._fitted: - return {} - result = {} - for cert_id, digests in self.id_mapping.items(): - cert_digest = digests[0] - cert_references = self.references[cert_digest] - direct_refs = cert_references["directly_referencing"] - if not direct_refs: - continue - unknowns = set(filter(lambda refd_cert_id: refd_cert_id not in self.id_mapping, direct_refs)) - if unknowns: - result[cert_id] = unknowns - return result - - @property - def duplicates(self) -> IDMapping: - """ - Get the duplicates in the fitted dataset. - - :return IDMapping: Mapping of certificate ID to digests that share it. - """ - if not self._fitted: - return {} - return {cert_id: digests for cert_id, digests in self.id_mapping.items() if len(digests) > 1} - - def predict_single_cert(self, dgst: str, keep_unknowns: bool = True) -> References: - """ - Get the references object for specified certificate digest. - - :param dgst: certificate digest - :param keep_unknowns: Whether to keep references to unknown certificate IDs - :return References: References object - """ - if not self._fitted: - raise ValueError("Finder not yet fitted") - - def wrap(res): - if not res: - return None - # If we do not want the unknown references, filter them here. - if not keep_unknowns: - res = set(filter(lambda cert_id: cert_id in self.id_mapping, res)) - return set(res) if res else None - - if dgst not in self.references: - return References() - - return References( - wrap(self.references[dgst].get("directly_referenced_by", None)), - wrap(self.references[dgst].get("indirectly_referenced_by", None)), - wrap(self.references[dgst].get("directly_referencing", None)), - wrap(self.references[dgst].get("indirectly_referencing", None)), - ) - - def predict(self, dgst_list: list[str], keep_unknowns: bool = True) -> dict[str, References]: - """ - Get the references for a list of certificate digests. - - :param dgst_list: List of certificate digests. - :param keep_unknowns: Whether to keep references to and from unknown certificate IDs - :return Dict[str, References]: Dict with certificate hash and References object. - """ - if not self._fitted: - raise ValueError("Finder not yet fitted") - cert_references = {} - - for dgst in dgst_list: - cert_references[dgst] = self.predict_single_cert(dgst, keep_unknowns=keep_unknowns) - - return cert_references diff --git a/src/sec_certs/model/references/segment_extractor.py b/src/sec_certs/model/references/segment_extractor.py deleted file mode 100644 index bf6096bc..00000000 --- a/src/sec_certs/model/references/segment_extractor.py +++ /dev/null @@ -1,390 +0,0 @@ -from __future__ import annotations - -import itertools -import json -import logging -import re -from dataclasses import dataclass -from importlib.resources import files -from pathlib import Path -from typing import Any, Iterable, Literal - -# import langdetect -import numpy as np -import pandas as pd -import spacy -from rapidfuzz import fuzz - -from sec_certs.sample.cc import CCCertificate -from sec_certs.sample.cc_certificate_id import CertificateId -from sec_certs.utils import parallel_processing - -nlp = spacy.load("en_core_web_sm") -logger = logging.getLogger(__name__) - - -def swap_and_filter_dict(dct: dict[str, Any], filter_to_keys: set[str]): - new_dct: dict[str, set[str]] = {} - for key, val in dct.items(): - if val in new_dct: - new_dct[val].add(key) - else: - new_dct[val] = {key} - - return {key: frozenset(val) for key, val in new_dct.items() if key in filter_to_keys} - - -def fill_reference_segments(record: ReferenceRecord, n_sent_before: int = 1, n_sent_after: int = 0) -> ReferenceRecord: - """ - Compute indices of the sentences containing the reference keyword, take their surrounding sentences and join them. - """ - - def compute_surroundings(hit_index: int, max_index: int, n_before: int, n_after: int): - """ - Computes indices of sentences to join into a coherent paragraph based on their location in text. - Ideally we would like to take (hit_index - n_before, hit_index + n_after), but we need to make sure - that we do not go out of bounds. - """ - lower = max(0, hit_index - n_before) - upper = min(max_index, hit_index + n_after) - return range(lower, upper + 1) - - with record.processed_data_source_path.open("r") as handle: - data = handle.read() - - sents = [sent.text for sent in nlp(data).sents] - indices_of_relevant_sents = [sents.index(x) for x in sents if any(y in x for y in record.actual_reference_keywords)] - - if not indices_of_relevant_sents: - record.segments = None - return record - - sequences_to_take = [ - compute_surroundings(x, len(sents) - 1, n_sent_before, n_sent_after) for x in indices_of_relevant_sents - ] - record.segments = {"".join([sents[y] for y in x]) for x in sequences_to_take} - - return record - - -def preprocess_data_source(record: ReferenceRecord) -> ReferenceRecord: - # TODO: There's some space for improvement, the preprocessing is acutally run twice. - - with record.raw_data_source_path.open("r") as handle: - data = handle.read() - - processed_data = preprocess_txt_func(data, record.actual_reference_keywords) - - with record.processed_data_source_path.open("w") as handle: - handle.write(processed_data) - - return record - - -def strip_all(text: str, to_strip) -> str: - if pd.isna(to_strip): - return text - for i in to_strip: - text = text.replace(i, "") - return text - - -def find_bracket_pattern(sentences: set[str], actual_reference_keywords: frozenset[str]): - patterns = [r"(\[.+?\])(?=.*" + x + r")" for x in actual_reference_keywords] - res: list[tuple[str, str]] = [] - - for sent in sentences: - for pattern, keyword in zip(patterns, actual_reference_keywords): - matches = re.findall(pattern, sent, flags=re.MULTILINE | re.UNICODE | re.DOTALL) - if matches: - res.append((matches[-1], keyword)) - return res - - -def preprocess_txt_func(data: str, actual_reference_keywords: frozenset[str]) -> str: - data = replace_acronyms(data) - data = replace_citation_identifiers(data, actual_reference_keywords) - return data - - -def replace_citation_identifiers(data: str, actual_reference_keywords: frozenset[str]) -> str: - segments = {sent.text for sent in nlp(data).sents if any(x in sent.text for x in actual_reference_keywords)} - patterns_to_replace = find_bracket_pattern(segments, actual_reference_keywords) - for x in patterns_to_replace: - data = data.replace(x[0], x[1]) - return data - - -def replace_acronyms(text: str) -> str: - acronym_replacements = { - "TOE": "target of evaluation", - "CC": "certification framework", - "PP": "protection profile", - "ST": "security target", - "SFR": "security Functional Requirement", - "SFRs": "security Functional Requirements", - "IC": "integrated circuit", - "MRTD": "machine readable travel document", - "TSF": "security functions of target of evaluation", - "PACE": "password authenticated connection establishment", - } - - for acronym, replacement in acronym_replacements.items(): - pattern = rf"(? tuple[str, str, frozenset[str], str, set[str] | None]: - return ( - self.certificate_dgst, - self.canonical_reference_keyword, - self.actual_reference_keywords, - self.source, - self.segments, - ) - - -class ReferenceSegmentExtractor: - """ - Class to process list of certificates into a dataframe that holds reference segments. - Should be only called with ReferenceSegmentExtractor()(list_of_certificates) - """ - - def __init__(self): - pass - - def __call__(self, certs: Iterable[CCCertificate]) -> pd.DataFrame: - return self._prepare_df_from_cc_dset(certs) - - def _prepare_df_from_cc_dset(self, certs: Iterable[CCCertificate]) -> pd.DataFrame: - """ - Prepares processed DataFrame for reference annotator training from a list of certificates. This method: - - Extracts text segments relevant for each reference out of the certificates, forms dataframe from those - - Loads data splits into train/valid/test (unseen certificates are put into test set) - - Loads manually annotated samples - - Combines all of that into single dataframe - """ - target_certs = [x for x in certs if x.heuristics.st_references.directly_referencing and x.state.st_txt_path] - report_certs = [ - x for x in certs if x.heuristics.report_references.directly_referencing and x.state.report_txt_path - ] - df_targets = self._build_df(target_certs, "target") - df_reports = self._build_df(report_certs, "report") - print(f"df_targets shape: {df_targets.shape}") - print(f"df_reports shape: {df_reports.shape}") - - return ReferenceSegmentExtractor._process_df(pd.concat([df_targets, df_reports]), certs) - - def _build_records(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> list[ReferenceRecord]: - def get_cert_records(cert: CCCertificate, source: Literal["target", "report"]) -> list[ReferenceRecord]: - canonical_ref_var = {"target": "st_references", "report": "report_references"} - actual_ref_var = {"target": "st_keywords", "report": "report_keywords"} - raw_source_var = {"target": "st_txt_path", "report": "report_txt_path"} - - canonical_references = getattr(cert.heuristics, canonical_ref_var[source]).directly_referencing - actual_references = getattr(cert.pdf_data, actual_ref_var[source])["cc_cert_id"] - actual_references = { - inner_key: CertificateId(outer_key, inner_key).canonical - for outer_key, val in actual_references.items() - for inner_key in val - } - actual_references = swap_and_filter_dict(actual_references, canonical_references) - - raw_source_dir = getattr(cert.state, raw_source_var[source]).parent - processed_source_dir = raw_source_dir.parent / "txt_processed" - - return [ - ReferenceRecord( - cert.dgst, - raw_source_dir / f"{cert.dgst}.txt", - processed_source_dir / f"{cert.dgst}.txt", - key, - val, - source, - ) - for key, val in actual_references.items() - ] - - (certs[0].state.report_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True) - (certs[0].state.st_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True) - return list(itertools.chain.from_iterable(get_cert_records(cert, source) for cert in certs)) - - def _build_df(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> pd.DataFrame: - records = self._build_records(certs, source) - - records = parallel_processing.process_parallel( - preprocess_data_source, - records, - use_threading=False, - progress_bar=True, - progress_bar_desc="Preprocessing data", - ) - - results = parallel_processing.process_parallel( - fill_reference_segments, - records, - use_threading=False, - progress_bar=True, - progress_bar_desc="Recovering reference segments", - ) - - print(f"I now have {len(results)} in {source} mode") - return pd.DataFrame.from_records( - [x.to_pandas_tuple() for x in results], - columns=["dgst", "canonical_reference_keyword", "actual_reference_keywords", "source", "segments"], - ) - - @staticmethod - def _get_split_dict() -> dict[str, str]: - """ - Returns dictionary that maps dgst: split, where split in `train`, `valid`, `test` - """ - - def get_single_dct(pth: Path, split_name: str) -> dict[str, str]: - with pth.open("r") as handle: - return dict.fromkeys(json.load(handle), split_name) - - split_directory = Path(str(files("sec_certs.data") / "reference_annotations/split/")) - return { - **get_single_dct(split_directory / "train.json", "train"), - **get_single_dct(split_directory / "valid.json", "valid"), - **get_single_dct(split_directory / "test.json", "test"), - } - - @staticmethod - def _get_annotations_dict() -> dict[tuple[str, str], str]: - """ - Returns dictionary mapping tuples `(dgst, canonical_reference_keyword) -> label` - """ - - def load_single_df(pth: Path, split_name: str) -> pd.DataFrame: - return ( - pd.read_csv( - pth, - na_values=["None"], - dtype={ - "dgst": str, - "canonical_reference_keyword": str, - "source": str, - "label": str, - "comment": str, - }, - ) - .dropna(subset="label") - .assign(label=lambda df_: df_.label.str.replace(" ", "_").str.upper(), split=split_name) - ) - - annotations_directory = Path(str(files("sec_certs.data") / "reference_annotations/final/")) - df_annot = pd.concat( - [ - load_single_df(annotations_directory / "train.csv", "train"), - load_single_df(annotations_directory / "valid.csv", "valid"), - load_single_df(annotations_directory / "test.csv", "test"), - ] - ) - - return ( - df_annot[["dgst", "canonical_reference_keyword", "label"]] - .set_index(["dgst", "canonical_reference_keyword"]) - .label.to_dict() - ) - - @staticmethod - def _process_df(df: pd.DataFrame, certs: Iterable[CCCertificate]) -> pd.DataFrame: - def process_segment(segment: str, actual_reference_keywords: frozenset[str]) -> str: - segment = " ".join(segment.split()) - for ref_id in actual_reference_keywords: - segment = segment.replace(ref_id, "REFERENCED_CERTIFICATE_ID") - return segment - - def unique_elements(series): - combined = [item for sublist in series for item in sublist] - return list(set(combined)) - - """ - Fully processes the dataframe. - """ - annotations_dict = ReferenceSegmentExtractor._get_annotations_dict() - split_dct = ReferenceSegmentExtractor._get_split_dict() - - # Retrieve some columns previously lost - dgst_to_cert_name = {x.dgst: x.name for x in certs} - cert_id_to_cert_name = {x.heuristics.cert_id: x.name for x in certs} - dgst_to_extracted_versions = {x.dgst: x.heuristics.extracted_versions for x in certs} - cert_id_to_extracted_versions = {x.heuristics.cert_id: x.heuristics.extracted_versions for x in certs} - - logger.info(f"Deleting {df.loc[df.segments.isnull()].shape[0]} rows with no segments.") - - df_new = df.copy() - df_new["full_key"] = df_new.apply(lambda x: (x["dgst"], x["canonical_reference_keyword"]), axis=1) - to_delete = len(df_new.loc[df_new.segments.isnull()].full_key.unique()) - print( - f"Deleting records for {to_delete} unique (dgst, referenced_id) pairs, not necessarily labeled ones. These have empty segments." - ) - - df_processed = ( - df.loc[df.segments.notnull()] - .explode("segments") - # .assign(lang=lambda df_: df_.segments.map(langdetect.detect)) - # .loc[lambda df_: df_.lang.isin({"en", "fr", "de"})] # This could get disabled possibly. - .groupby( - ["dgst", "canonical_reference_keyword"], - as_index=False, - dropna=False, - ) - .agg({"segments": list, "actual_reference_keywords": unique_elements}) - .assign( - split=lambda df_: df_.dgst.map(split_dct), - label=lambda df_: [ - annotations_dict.get(x) for x in zip(df_["dgst"], df_["canonical_reference_keyword"]) - ], - actual_reference_keywords=lambda df_: df_.actual_reference_keywords.map(list), - cert_name=lambda df_: df_.dgst.map(dgst_to_cert_name), - referenced_cert_name=lambda df_: df_.canonical_reference_keyword.map(cert_id_to_cert_name), - cert_versions=lambda df_: df_.dgst.map(dgst_to_extracted_versions), - referenced_cert_versions=lambda df_: df_.canonical_reference_keyword.map(cert_id_to_extracted_versions), - cert_name_stripped_version=lambda df_: df_.apply( - lambda x: strip_all(x["cert_name"], x["cert_versions"]), axis=1 - ), - referenced_cert_name_stripped_version=lambda df_: df_.apply( - lambda x: strip_all(x["referenced_cert_name"], x["referenced_cert_versions"]), axis=1 - ), - name_similarity=lambda df_: df_.apply( - lambda x: fuzz.token_set_ratio( - x["cert_name_stripped_version"], x["referenced_cert_name_stripped_version"] - ), - axis=1, - ), - name_len_diff=lambda df_: df_.apply( - lambda x: np.nan - if pd.isnull(x["cert_name_stripped_version"]) - or pd.isnull(x["referenced_cert_name_stripped_version"]) - else abs(len(x["cert_name_stripped_version"]) - len(x["referenced_cert_name_stripped_version"])), - axis=1, - ), - ) - .assign( - label=lambda df_: df_.label.map(lambda x: x if x is not None else np.nan), - split=lambda df_: df_.split.map(lambda x: "test" if pd.isnull(x) else x), - ) - ) - df_processed.segments = df_processed.apply( - lambda row: [process_segment(x, row.actual_reference_keywords) for x in row.segments], axis=1 - ) - return df_processed diff --git a/src/sec_certs/model/references_nlp/__init__.py b/src/sec_certs/model/references_nlp/__init__.py new file mode 100644 index 00000000..82c95aa1 --- /dev/null +++ b/src/sec_certs/model/references_nlp/__init__.py @@ -0,0 +1,13 @@ +# ruff: noqa: F401 +try: + import catboost + import optuna + import plotly.express + import setfit + import sklearn + import umap +except ImportError as e: + print(e) + print( + f"Requirements for ML annotation of references not met. Please run `pip install sec-certs[nlp]` or install `pip install -r requirements/nlp_requirements.txt." + ) diff --git a/src/sec_certs/model/references_nlp/annotator.py b/src/sec_certs/model/references_nlp/annotator.py new file mode 100644 index 00000000..1cc816f5 --- /dev/null +++ b/src/sec_certs/model/references_nlp/annotator.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import json +import logging +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from setfit import SetFitModel + +from sec_certs.utils.nlp import softmax + +logger = logging.getLogger(__name__) + + +@dataclass +class ReferenceAnnotator: + """ + Class for annotating references. Its instances are supposed to by trained by `ReferenceAnnotatorTrainer`. + Can be serialized into a directory / load from a directory. + """ + + _model: Any + _label_mapping: dict[int, str] + _soft_voting_power: int = 2 + + @classmethod + def from_pretrained(cls, model_dir: str | Path) -> ReferenceAnnotator: + """ + Loads classifier from directory, assuming that: + - the SetFitModel was dumped into that directory with model.save_pretrained(model_dir) + - json file label_mapping.json exists in model_dir + + :param str | Path model_dir: path to directory to search for model and label mapping + :return RerefenceClassifier: classifier with SetFitModel and label mapping + """ + logger.info(f"Loading pre-trained reference annotator from: {model_dir}") + model = SetFitModel.from_pretrained(str(model_dir)) + with (Path(model_dir) / "label_mapping.json").open("r") as handle: + label_mapping = json.load(handle) + label_mapping = {int(k): v for k, v in label_mapping.items()} + + return cls(model, label_mapping) + + def save_pretrained(self, model_dir: str | Path): + """ + Will dump _model and _label_mapping into a directory. + """ + logger.info(f"Saving ReferenceAnnotator to {model_dir}") + model_dir = Path(model_dir) + model_dir.mkdir(exist_ok=True, parents=True) + logger.info + with (model_dir / "label_mapping.json").open("w") as handle: + json.dump(self._label_mapping, handle, indent=4) + self._model._save_pretrained(str(model_dir)) + + def train(self, train_dataset: pd.DataFrame): + raise NotImplementedError("ReferenceAnnotatorTrainer shall be used for training") + + def predict(self, X: list[list[str]]) -> list[str]: + return [self._predict_single(x) for x in X] + + def _predict_single(self, sample: list[str]) -> str: + # return self._predict_single_majority_vote(sample) + return self._label_mapping[int(np.argmax(self._predict_proba_single(sample)))] + + def predict_proba(self, X: list[list[str]]) -> list[list[float]]: + return [self._predict_proba_single(x) for x in X] + + def _predict_proba_single(self, sample: list[str]) -> list[float]: + """ + 1. Get predictions for each segment, convert pytorch tensor to numpy + 2. Square every prediction to reward confidence + 3. Sum probabilities for each label + 4. softmax + """ + return softmax(np.power(self._model.predict_proba(sample, as_numpy=True), self._soft_voting_power).sum(axis=0)) + + def _predict_single_majority_vote(self, sample: list[str]) -> str: + predictions = self._model.predict(sample) + most_common = Counter(predictions).most_common() + if len(most_common) > 1 and most_common[0][1] == most_common[1][1]: + return self._label_mapping[int(np.argmax(self._predict_proba_single(sample)))] + else: + return self._label_mapping[int(most_common[0][0])] + + def predict_df(self, df: pd.DataFrame) -> pd.DataFrame: + """ + WIll read df.segments and populate the dataframe with predictions. + """ + df_new = df.copy() + y_proba = self.predict_proba(df_new.segments) + df_new["y_proba"] = y_proba + df_new["y_pred"] = self.predict(df_new.segments) + df_new["correct"] = df_new.apply( + lambda row: row["y_pred"] == row["label"] if not pd.isnull(row["label"]) else np.NaN, axis=1 + ) + return df_new diff --git a/src/sec_certs/model/references_nlp/annotator_trainer.py b/src/sec_certs/model/references_nlp/annotator_trainer.py new file mode 100644 index 00000000..1f973842 --- /dev/null +++ b/src/sec_certs/model/references_nlp/annotator_trainer.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import logging +from functools import partial +from typing import Callable, Final, Literal + +import pandas as pd +from datasets import ClassLabel, Dataset, Features, NamedSplit, Value +from sentence_transformers.losses import CosineSimilarityLoss +from setfit import SetFitModel, SetFitTrainer +from sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score + +from sec_certs.model.references_nlp.annotator import ReferenceAnnotator +from sec_certs.utils.nlp import prepare_reference_annotations_df + +logger = logging.getLogger(__name__) + + +class ReferenceAnnotatorTrainer: + METRIC_TO_USE: Final[dict[str, Callable]] = { + "accuracy": accuracy_score, + "balanced_accuracy": balanced_accuracy_score, + "f1": partial(f1_score, average="weighted", zero_division=0), + } + + def __init__( + self, + train_dataset: pd.DataFrame, + eval_dataset: pd.DataFrame, + metric: Callable, + n_iterations: int = 20, + learning_rate: float = 2e-5, + n_epochs: int = 1, + batch_size: int = 16, + segmenter_metric: Literal["accuracy", "f1", "balanced_accuracy"] = "accuracy", + ensemble_soft_voting_power: int = 2, + show_progress_bar: bool = True, + ): + self._train_dataset = train_dataset + self._eval_dataset = eval_dataset + self._metric = metric + self.n_iterations = n_iterations + self.learning_rate = learning_rate + self.n_epochs = n_epochs + self.batch_size = batch_size + self.segmenter_metric = segmenter_metric + self.ensemble_soft_voting_power = ensemble_soft_voting_power + self.show_progress_bar = show_progress_bar + + self._model, self._trainer, self.label_mapping = self._init_trainer() + + self.clf = ReferenceAnnotator( + self._model, + self.label_mapping, + self.ensemble_soft_voting_power, + ) + + @classmethod + def from_df( + cls, + df: pd.DataFrame, + metric: Callable, + mode: Literal["training", "evaluation", "production", "cross-validation"] = "training", + n_iterations: int = 20, + learning_rate: float = 2e-5, + n_epochs: int = 1, + batch_size: int = 16, + segmenter_metric: Literal["accuracy", "f1", "balanced_accuracy"] = "accuracy", + ensemble_soft_voting_power: int = 2, + show_progress_bar: bool = True, + ): + df = prepare_reference_annotations_df(df) + dataset_generation_method = { + "training": ReferenceAnnotatorTrainer.split_df_for_training, + "evaluation": ReferenceAnnotatorTrainer.split_df_for_evaluation, + "production": ReferenceAnnotatorTrainer.split_df_for_production, + "cross-validation": ReferenceAnnotatorTrainer.split_df_for_training, + } + + train_dataset, eval_dataset = dataset_generation_method[mode](df) + return cls( + train_dataset, + eval_dataset, + metric, + n_iterations, + learning_rate, + n_epochs, + batch_size, + segmenter_metric, + ensemble_soft_voting_power, + show_progress_bar, + ) + + @staticmethod + def split_df_for_training(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + return df.loc[df.split == "train"].drop(columns="split"), df.loc[df.split == "valid"].drop(columns="split") + + @staticmethod + def split_df_for_production(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + logger.info("Splitting dataset for production, model can be trained, but not evaluated.") + return df.drop(columns="split"), df.drop(df.index) + + @staticmethod + def split_df_for_evaluation(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + df.split = df.split.map({"test": "test", "train": "train", "valid": "train"}) + if df.loc[df.split == "test"].empty: + logger.warning("`test` split for annotator dataset is empty -> model can be trained, but not evaluated.") + return df.loc[df.split == "train"].drop(columns="split"), df.loc[df.split == "test"].drop(columns="split") + + def _init_trainer(self): + model = SetFitModel.from_pretrained("paraphrase-multilingual-mpnet-base-v2") + # model = SetFitModel.from_pretrained("all-mpnet-base-v2") + + train_dataset_relevant_cols = self._train_dataset[["dgst", "canonical_reference_keyword", "segments", "label"]] + eval_dataset_relevant_cols = self._eval_dataset[["dgst", "canonical_reference_keyword", "segments", "label"]] + internal_train_dataset = self._get_hugging_face_datasets_from_df(train_dataset_relevant_cols, "train") + internal_validation_dataset = self._get_hugging_face_datasets_from_df(eval_dataset_relevant_cols, "validation") + + # Align labels alphabetically + labels_alphabetically = sorted(internal_train_dataset.features["label"].names) + label2id = {label: index for index, label in enumerate(labels_alphabetically)} + internal_train_dataset = internal_train_dataset.align_labels_with_mapping(label2id, "label") + internal_validation_dataset = internal_validation_dataset.align_labels_with_mapping(label2id, "label") + + trainer = SetFitTrainer( + model=model, + train_dataset=internal_train_dataset, + eval_dataset=internal_validation_dataset, + loss_class=CosineSimilarityLoss, + metric=self.METRIC_TO_USE[self.segmenter_metric], + learning_rate=self.learning_rate, + batch_size=self.batch_size, + num_iterations=self.n_iterations, # The number of text pairs to generate for contrastive learning + num_epochs=self.n_epochs, # The number of epochs to use for contrastive learning + column_mapping={ + "segment": "text", + "label": "label", + }, # Map dataset columns to text/label expected by trainer + ) + return model, trainer, {index: label for label, index in label2id.items()} + + @staticmethod + def _get_hugging_face_datasets_from_df(df: pd.DataFrame, split: NamedSplit) -> Dataset: + df_to_use = df.explode("segments").rename(columns={"segments": "segment"}).loc[df.label.notnull()] + features = Features( + { + "dgst": Value("string"), + "canonical_reference_keyword": Value("string"), + "segment": Value("string"), + "label": ClassLabel(names=list(df_to_use.label.unique())), + } + ) + return Dataset.from_pandas(df_to_use, features=features, split=split, preserve_index=False) + + def train(self): + self._trainer.train(show_progress_bar=self.show_progress_bar) + + def evaluate(self): + print("Internal evaluation (of model working on individual segments)") + print(self._evaluate_raw()) + print("Actual evaluation after ensemble soft voting") + print(self._evaluate_stacked()) + + def _evaluate_raw(self): + if self._eval_dataset.empty: + logger.error("Evaluation dataset is empty, cannot evaluate, returning.") + return + return self._trainer.evaluate() + + def _evaluate_stacked(self): + y_pred = self.clf.predict(self._eval_dataset.segments) + y_true = self._eval_dataset.label + return self._metric(y_pred, y_true) diff --git a/src/sec_certs/model/references_nlp/evaluation.py b/src/sec_certs/model/references_nlp/evaluation.py new file mode 100644 index 00000000..dc9b4b13 --- /dev/null +++ b/src/sec_certs/model/references_nlp/evaluation.py @@ -0,0 +1,90 @@ +import logging +from pathlib import Path +from typing import Literal + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import plotly.express as px +from catboost import CatBoostClassifier +from sklearn.dummy import DummyClassifier +from sklearn.metrics import ConfusionMatrixDisplay, balanced_accuracy_score, classification_report + +logger = logging.getLogger(__name__) + + +def evaluate_model( + clf: DummyClassifier | CatBoostClassifier, + df_eval: pd.DataFrame, + feature_cols: list[str], + output_path: Path | None = None, +): + logger.info("Evaluating model.") + x_eval = np.vstack(df_eval[feature_cols].values) + y_pred = clf.predict(x_eval) + + df_eval["y_pred"] = y_pred + + df_eval.loc[df_eval.lang_matches_recertification, ["y_pred"]] = "PREVIOUS_VERSION" + df_eval.loc[ + (df_eval.lang_token_set_ratio == 100) + & (df_eval.lang_len_difference < 5) + & (df_eval.y_pred != "PREVIOUS_VERSION"), + ["y_pred"], + ] = "PREVIOUS_VERSION" + + print(classification_report(df_eval.label.values, df_eval.y_pred.values)) + print(f"Balanced accuracy score: {balanced_accuracy_score(df_eval.label.values, df_eval.y_pred.values)}") + + fig = ConfusionMatrixDisplay.from_predictions( + df_eval.label.values, + df_eval.y_pred.values, + xticks_rotation=90, + ) + + if output_path: + report_dict = classification_report(df_eval.label.values, df_eval.y_pred.values, output_dict=True) + report_df = pd.DataFrame(report_dict).transpose() + report_df.to_csv(output_path / "classification_report.csv") + fig.figure_.savefig(output_path / "confusion_matrix.png") + with Path(output_path / "balanced_accuracy_score.txt").open("w") as handle: + handle.write(str(balanced_accuracy_score(df_eval.label.values, df_eval.y_pred.values))) + + if isinstance(clf, CatBoostClassifier): + feature_importance = clf.get_feature_importance() + sorted_idx = np.argsort(feature_importance) + features = np.array(feature_cols)[sorted_idx] + + fig_feature_importance = plt.figure(figsize=(10, 12)) + plt.barh(features, feature_importance[sorted_idx], align="center") + plt.xlabel("Feature Importance") + plt.ylabel("Feature") + plt.title("Feature Importance in Gradient boosted trees classifier") + plt.tight_layout() + plt.show() + + if output_path: + fig_feature_importance.savefig(output_path / "feature_importance.png") + + +def display_dim_red_scatter(df: pd.DataFrame, dim_red: Literal["umap", "pca"]) -> None: + df_exploded = df.explode(["segments", dim_red]).reset_index() + + x_col = dim_red + "_x" + y_col = dim_red + "_y" + + df_exploded[x_col] = df_exploded[dim_red].map(lambda x: x[0]) + df_exploded[y_col] = df_exploded[dim_red].map(lambda x: x[1]) + df_exploded["wrapped_segment"] = df_exploded.segments.str.wrap(60).map(lambda x: x.replace("\n", "
")) + + fig = px.scatter( + df_exploded, + x=x_col, + y=y_col, + color="label", + hover_data=["dgst", "canonical_reference_keyword", "wrapped_segment"], + width=1500, + height=1000, + title=f"{dim_red.upper()} projection of segment embeddings.", + ) + fig.show() diff --git a/src/sec_certs/model/references_nlp/feature_extraction.py b/src/sec_certs/model/references_nlp/feature_extraction.py new file mode 100644 index 00000000..eed90959 --- /dev/null +++ b/src/sec_certs/model/references_nlp/feature_extraction.py @@ -0,0 +1,542 @@ +import itertools +import logging +import re +from collections import Counter +from pathlib import Path +from typing import Literal + +import numpy as np +import pandas as pd +import spacy +import umap +import umap.plot +from rapidfuzz import fuzz +from scipy.spatial import ConvexHull, QhullError, distance_matrix +from scipy.stats import kurtosis, skew +from sklearn.decomposition import PCA +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.preprocessing import LabelEncoder, StandardScaler + +from sec_certs.constants import RANDOM_STATE, REF_ANNOTATION_MODES, REF_EMBEDDING_METHOD +from sec_certs.dataset import CCDataset +from sec_certs.model.references_nlp.annotator import ReferenceAnnotator +from sec_certs.model.references_nlp.annotator_trainer import ReferenceAnnotatorTrainer +from sec_certs.model.references_nlp.segment_extractor import ReferenceSegmentExtractor +from sec_certs.utils.nlp import prec_recall_metric + +logger = logging.getLogger(__name__) + +nlp = spacy.load("en_core_web_sm") + + +def strip_all(text: str, to_strip) -> str: + if pd.isna(to_strip): + return text + for i in to_strip: + text = text.replace(i, "") + return text + + +def matches_recertification(segments: list[str]) -> bool: + regex_a = r"This is a re-?\s?certification based on (the\s){0,1}REFERENCED_CERTIFICATE_ID" + regex_b = r"Re-?\s?Zertifizierung basierend auf (the\s){0,1}REFERENCED_CERTIFICATE_ID" + return any( + re.search(regex_a, segment, re.IGNORECASE) or re.search(regex_b, segment, re.IGNORECASE) for segment in segments + ) + + +def compute_ngram_overlap_spacy(string1, string2, n): + doc1 = nlp(string1) + doc2 = nlp(string2) + + ngrams1 = [" ".join([token.text for token in doc1[i : i + n]]) for i in range(len(doc1) - n + 1)] + ngrams2 = [" ".join([token.text for token in doc2[i : i + n]]) for i in range(len(doc2) - n + 1)] + + overlap = sum((Counter(ngrams1) & Counter(ngrams2)).values()) + return overlap + + +def compute_character_ngram_overlap(str1, str2, n): + ngrams1 = [str1[i : i + n] for i in range(len(str1) - n + 1)] + ngrams2 = [str2[i : i + n] for i in range(len(str2) - n + 1)] + overlap = sum((Counter(ngrams1) & Counter(ngrams2)).values()) + return overlap + + +def compute_common_length(str1, str2, prefix=True): + length = 0 + min_length = min(len(str1), len(str2)) + if prefix: + for i in range(min_length): + if str1[i] == str2[i]: + length += 1 + else: + break + else: + for i in range(1, min_length + 1): + if str1[-i] == str2[-i]: + length += 1 + else: + break + return length + + +def compute_numeric_token_overlap(str1, str2): + doc1 = nlp(str1) + doc2 = nlp(str2) + + tokens1 = [token.text for token in doc1 if token.like_num] + tokens2 = [token.text for token in doc2 if token.like_num] + + overlap = sum((Counter(tokens1) & Counter(tokens2)).values()) + return overlap + + +def get_lang_features(base_name: str, referenced_name: str) -> tuple: + common_numeric_words = compute_numeric_token_overlap(base_name, referenced_name) + common_words = compute_ngram_overlap_spacy(base_name, referenced_name, 1) + bigram_overlap = compute_ngram_overlap_spacy(base_name, referenced_name, 2) + trigram_overlap = compute_ngram_overlap_spacy(base_name, referenced_name, 3) + common_prefix_len = compute_common_length(base_name, referenced_name, True) + common_suffix_len = compute_common_length(base_name, referenced_name, False) + character_bigram_overlap = compute_character_ngram_overlap(base_name, referenced_name, 2) + character_trigram_overlap = compute_character_ngram_overlap(base_name, referenced_name, 3) + base_len = len(base_name) + referenced_len = len(referenced_name) + len_difference = abs(base_len - referenced_len) + + return ( + common_numeric_words, + common_words, + bigram_overlap, + trigram_overlap, + common_prefix_len, + common_suffix_len, + character_bigram_overlap, + character_trigram_overlap, + base_len, + referenced_len, + len_difference, + ) + + +def extract_segments( + cc_dset: CCDataset, mode: REF_ANNOTATION_MODES, n_sents_before: int = 2, n_sents_after: int = 1 +) -> pd.DataFrame: + logger.info("Extracting segments.") + df = ReferenceSegmentExtractor(n_sents_before, n_sents_after)(list(cc_dset.certs.values())) + if mode == "training": + return df.loc[(df.label.notnull()) & ((df.split == "train") | (df.split == "valid"))] + elif mode == "evaluation": + return df.loc[df.label.notnull()] + elif mode == "production": + return df + else: + raise ValueError(f"Unknown mode {mode}") + + +def _build_transformer_embeddings( + segments: pd.DataFrame, mode: REF_ANNOTATION_MODES, model_path: Path | None = None +) -> pd.DataFrame: + should_save_model = model_path is not None + annotator = None + logger.info("Building transformer embeddings.") + if model_path: + try: + annotator = ReferenceAnnotator.from_pretrained(model_path) + should_save_model = False + except Exception: + print(f"Failed to load ReferenceAnnotator from {model_path}.") + should_save_model = True + + if not annotator: + print("Training ReferenceAnnotator from scratch.") + trainer = ReferenceAnnotatorTrainer.from_df( + segments, + prec_recall_metric, + mode=mode, + n_iterations=34, + n_epochs=1, + learning_rate=0.01, + batch_size=16, + segmenter_metric="f1", + ensemble_soft_voting_power=2, + show_progress_bar=False, + ) + trainer.train() + annotator = trainer.clf + assert annotator is not None + + if should_save_model and model_path: + annotator.save_pretrained(model_path) + + return ( + segments.copy().assign(embeddings=lambda df_: df_.segments.map(annotator._model.model_body.encode)), + annotator, + ) + + +def _build_tf_idf_embeddings(segments: pd.DataFrame, mode: REF_ANNOTATION_MODES) -> pd.DataFrame: + def choose_values_to_fit(df_: pd.DataFrame) -> list[str]: + if mode == "training": + return df_.loc[df_.split == "train"].copy().explode("segments").segments.values + elif mode == "evaluation": + return df_.loc[df_.split != "test"].copy().explode("segments").segments.values + elif mode == "production": + return df_.copy().explode("segments").segments.values + else: + raise ValueError(f"Unknown mode {mode}") + + logger.info("Building TF-IDF embeddings.") + tf_idf = TfidfVectorizer() + tf_idf = tf_idf.fit(choose_values_to_fit(segments)) + + return segments.copy().assign( + embeddings=lambda df_: df_.segments.map(lambda x: tf_idf.transform(x).toarray().tolist()) + ) + + +def build_embeddings( + segments: pd.DataFrame, mode: REF_ANNOTATION_MODES, method: REF_EMBEDDING_METHOD, model_path: Path | None = None +) -> pd.DataFrame: + return ( + _build_transformer_embeddings(segments, mode, model_path) + if method == "transformer" + else _build_tf_idf_embeddings(segments, mode) + ) + + +def extract_language_features(df: pd.DataFrame, cc_dset: CCDataset) -> pd.DataFrame: + logger.info("Extracting language features.") + certs = list(cc_dset.certs.values()) + dgst_to_cert_name = {x.dgst: x.name for x in certs} + cert_id_to_cert_name = {x.heuristics.cert_id: x.name for x in certs} + dgst_to_extracted_versions = {x.dgst: x.heuristics.extracted_versions for x in certs} + cert_id_to_extracted_versions = {x.heuristics.cert_id: x.heuristics.extracted_versions for x in certs} + + df_lang = ( + df.copy() + .assign( + cert_name=lambda df_: df_.dgst.map(dgst_to_cert_name), + referenced_cert_name=lambda df_: df_.canonical_reference_keyword.map(cert_id_to_cert_name), + cert_versions=lambda df_: df_.dgst.map(dgst_to_extracted_versions), + referenced_cert_versions=lambda df_: df_.canonical_reference_keyword.map(cert_id_to_extracted_versions), + cert_name_stripped_version=lambda df_: df_.apply( + lambda x: strip_all(x["cert_name"], x["cert_versions"]), axis=1 + ), + referenced_cert_name_stripped_version=lambda df_: df_.apply( + lambda x: strip_all(x["referenced_cert_name"], x["referenced_cert_versions"]), axis=1 + ), + lang_token_set_ratio=lambda df_: df_.apply( + lambda x: fuzz.token_set_ratio( + x["cert_name_stripped_version"], x["referenced_cert_name_stripped_version"] + ), + axis=1, + ), + lang_partial_ratio=lambda df_: df_.apply( + lambda x: fuzz.partial_ratio( + x["cert_name_stripped_version"], x["referenced_cert_name_stripped_version"] + ), + axis=1, + ), + lang_token_sort_ratio=lambda df_: df_.apply( + lambda x: fuzz.token_sort_ratio( + x["cert_name_stripped_version"], x["referenced_cert_name_stripped_version"] + ), + axis=1, + ), + lang_n_segments=lambda df_: df_.segments.map(lambda x: len(x) if x else 0), + lang_matches_recertification=lambda df_: df_.segments.map(matches_recertification), + ) + .assign( + lang_n_extracted_versions=lambda df_: df_.cert_versions.map(lambda x: len(x) if x else 0), + lang_n_intersection_versions=lambda df_: df_.apply( + lambda x: len(set(x["cert_versions"]).intersection(set(x["referenced_cert_versions"]))), axis=1 + ), + ) + ) + + df_lang_other_features = df_lang.apply( + lambda row: get_lang_features(row["cert_name"], row["referenced_cert_name"]), axis=1 + ).apply(pd.Series) + lang_features = [ + "common_numeric_words", + "common_words", + "bigram_overlap", + "trigram_overlap", + "common_prefix_len", + "common_suffix_len", + "character_bigram_overlap", + "character_trigram_overlap", + "base_len", + "referenced_len", + "len_difference", + ] + df_lang_other_features.columns = ["lang_" + x for x in lang_features] + + df_lang = pd.concat([df_lang, df_lang_other_features], axis=1).assign( + lang_should_not_be_component=lambda df_: df_.apply( + lambda x: x.lang_len_difference < 5 and x.lang_token_set_ratio == 100, axis=1 + ), + ) + for col in df_lang.columns: + if col.startswith("pred_"): + df_lang[col] = df_lang[col] / df_lang.lang_n_segments + + return df_lang + + +def perform_dimensionality_reduction( + df: pd.DataFrame, + mode: REF_ANNOTATION_MODES, + umap_n_neighbors: int = 5, + umap_min_dist: float = 0.1, + umap_metric: Literal["cosine", "euclidean", "manhattan"] = "euclidean", +) -> pd.DataFrame: + def choose_values_to_fit(df_: pd.DataFrame): + if mode == "training": + return df_.loc[df_.split == "train"].copy().embeddings.values + elif mode == "evaluation": + return df_.loc[df_.split != "test"].copy().embeddings.values + elif mode == "production": + return df_.copy().embeddings.values + else: + raise ValueError(f"Unknown mode {mode}") + + def choose_labels_to_fit(df_: pd.DataFrame): + if mode == "training": + return df_.loc[df_.split == "train"].copy().label.values + elif mode == "evaluation": + return df_.loc[df_.split != "test"].copy().label.values + elif mode == "production": + return df_.copy().label.values + else: + raise ValueError(f"Unknown mode {mode}") + + logger.info("Performing dimensionality reduction.") + df_exploded = df.copy().explode(["segments", "embeddings"]).reset_index(drop=True) + label_encoder = LabelEncoder() + + embeddings_to_fit = np.vstack(choose_values_to_fit(df_exploded)) + labels_to_fit = label_encoder.fit_transform(choose_labels_to_fit(df_exploded)) + + scaler = StandardScaler() + embeddings_to_fit_scaled = scaler.fit_transform(embeddings_to_fit) + + # parallel UMAP not available with random state + umapper = umap.UMAP( + n_neighbors=umap_n_neighbors, min_dist=umap_min_dist, metric=umap_metric, random_state=RANDOM_STATE, n_jobs=1 + ).fit(embeddings_to_fit, y=labels_to_fit) + pca_mapper = PCA(n_components=2, random_state=RANDOM_STATE).fit(embeddings_to_fit_scaled, y=labels_to_fit) + + all_embeddings = np.vstack(df.embeddings.values) + all_embeddings_scaled = scaler.transform(all_embeddings) + + df_exploded["umap"] = umapper.transform(all_embeddings).tolist() + df_exploded["pca"] = pca_mapper.transform(all_embeddings_scaled).tolist() + + return ( + df_exploded.groupby(["dgst", "canonical_reference_keyword"]) + .agg( + { + "segments": lambda x: x.tolist(), + "actual_reference_keywords": "first", + "label": "first", + "split": "first", + "embeddings": lambda x: x.tolist(), + "umap": lambda x: x.tolist(), + "pca": lambda x: x.tolist(), + } + ) + .reset_index() + ) + + +def extract_prediction_features(df: pd.DataFrame, model) -> pd.DataFrame: + def get_setfit_prediction_numbers(val): + counter = Counter(val.tolist()) + return [counter[x] for x in range(len(all_labels))] + + logger.info("Extracting prediction features.") + df["annotator_predictions"] = df.segments.map(lambda x: model.predict(x)) + all_labels = set(itertools.chain.from_iterable(x.tolist() for x in df.annotator_predictions.values)) + + df_features_pred = df.annotator_predictions.apply(get_setfit_prediction_numbers).apply(pd.Series) + feature_names = [f"pred_{x}" for x in range(len(all_labels))] + df_features_pred.columns = feature_names + return pd.concat([df, df_features_pred], axis=1) + + +def extract_geometrical_features(df: pd.DataFrame) -> pd.DataFrame: + def extract_features(points): + # Convert list of points to a numpy array + points = np.array(points) + xs = points[:, 0] + ys = points[:, 1] + + # Basic Descriptive Statistics + mean_x, mean_y = np.mean(xs), np.mean(ys) + var_x, var_y = np.var(xs), np.var(ys) + std_x, std_y = np.std(xs), np.std(ys) + if len(points) > 1: + skew_x, skew_y = skew(xs), skew(ys) + kurt_x, kurt_y = kurtosis(xs), kurtosis(ys) + else: + skew_x, skew_y = 0, 0 + kurt_x, kurt_y = 0, 0 + + # Spatial Spread + range_x, range_y = np.ptp(xs), np.ptp(ys) + cov_xy = np.cov(xs, ys)[0, 1] if len(points) > 1 else 0 + median_x, median_y = np.median(xs), np.median(ys) + + # Distance-based Features + centroid = [mean_x, mean_y] + distances_to_centroid = np.linalg.norm(points - centroid, axis=1) if len(points) > 1 else [0] + mean_distance = np.mean(distances_to_centroid) + max_distance = np.max(distances_to_centroid) + min_distance = np.min(distances_to_centroid) + std_distance = np.std(distances_to_centroid) + max_min_distance = max_distance - min_distance + + sorted_points = points[np.argsort(distances_to_centroid)] + total_distance = np.sum(np.linalg.norm(sorted_points[1:] - sorted_points[:-1], axis=1)) + + # Geometric Features + hull_area, hull_perimeter = (0, 0) + if len(points) > 2: # ConvexHull needs at least 3 points + try: + hull = ConvexHull(points) + hull_area = hull.volume + hull_perimeter = hull.area + except QhullError: + pass + + pairwise_distances = distance_matrix(points, points) if len(points) > 1 else np.array([[0]]) + mean_pairwise_distance = np.mean(pairwise_distances) + max_pairwise_distance = np.max(pairwise_distances) + + if len(points) > 1: + min_coords = np.min(points, axis=0) + max_coords = np.max(points, axis=0) + bounding_box_width = max_coords[0] - min_coords[0] + bounding_box_height = max_coords[1] - min_coords[1] + bounding_box_area = bounding_box_width * bounding_box_height + + aspect_ratio = bounding_box_width / bounding_box_height if bounding_box_height != 0 else 1 + point_density = len(points) / bounding_box_area + else: + aspect_ratio = 0 + point_density = 0 + + # Gather all features into a list + features = [ + mean_x, + mean_y, + var_x, + var_y, + std_x, + std_y, + skew_x, + skew_y, + kurt_x, + kurt_y, + range_x, + range_y, + cov_xy, + median_x, + median_y, + mean_distance, + max_distance, + min_distance, + max_min_distance, + std_distance, + total_distance, + hull_area, + hull_perimeter, + mean_pairwise_distance, + max_pairwise_distance, + aspect_ratio, + point_density, + ] + + return features + + feature_names = [ + "mean_x", + "mean_y", + "var_x", + "var_y", + "std_x", + "std_y", + "skew_x", + "skew_y", + "kurt_x", + "kurt_y", + "range_x", + "range_y", + "cov_xy", + "median_x", + "median_y", + "mean_distance_to_centroid", + "max_distance_to_centroid", + "min_distance_to_centroid", + "max_min_distance_to_centroid", + "std_distance_to_centroid", + "total_distances_to_centroid", + "hull_area", + "hull_perimeter", + "mean_pairwise_distance", + "max_pairwise_distance", + "aspect_ratio", + "point_density", + ] + + logger.info("Extracting geometrical features.") + df_features_pca = df.pca.apply(extract_features).apply(pd.Series) + feature_names_pca = ["pca_" + x for x in feature_names] + df_features_pca.columns = feature_names_pca + + df_features_umap = df.umap.apply(extract_features).apply(pd.Series) + feature_names_umap = ["umap_" + x for x in feature_names] + df_features_umap.columns = feature_names_umap + + return pd.concat([df, df_features_pca, df_features_umap], axis=1) + + +def get_data_for_clf( + df: pd.DataFrame, + mode: REF_ANNOTATION_MODES, + use_pca: bool = True, + use_umap: bool = True, + use_lang: bool = True, + use_pred: bool = True, +) -> tuple[np.ndarray, np.ndarray, pd.DataFrame | None, list[str]]: + feature_columns = [] + if not use_pca and not use_umap and not use_lang and not use_pred: + raise ValueError("At least one of PCA, UMAP or language features must be used.") + if use_pca: + feature_columns.extend([x for x in df.columns if x.startswith("pca_")]) + if use_umap: + feature_columns.extend([x for x in df.columns if x.startswith("umap_")]) + if use_lang: + feature_columns.extend([x for x in df.columns if x.startswith("lang_")]) + if use_pred: + feature_columns.extend([x for x in df.columns if x.startswith("pred_")]) + + if mode == "training": + train_df = df.loc[df.split == "train"].copy() + eval_df = df.loc[df.split == "valid"].copy() + elif mode == "evaluation": + train_df = df.loc[df.split != "test"].copy() + eval_df = df.loc[df.split == "test"].copy() + elif mode == "production": + train_df = df.copy() + eval_df = df.copy() + elif mode == "cross-validation": + train_df = df.loc[df.split != "test"].copy() + eval_df = None + else: + raise ValueError(f"Unknown mode {mode}") + + return np.vstack(train_df[feature_columns].values), train_df.label.values, eval_df, feature_columns diff --git a/src/sec_certs/model/references_nlp/segment_extractor.py b/src/sec_certs/model/references_nlp/segment_extractor.py new file mode 100644 index 00000000..ce144b93 --- /dev/null +++ b/src/sec_certs/model/references_nlp/segment_extractor.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import itertools +import json +import logging +import re +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from typing import Any, Iterable, Literal + +# import langdetect +import numpy as np +import pandas as pd +import spacy + +from sec_certs.sample.cc import CCCertificate +from sec_certs.sample.cc_certificate_id import CertificateId +from sec_certs.utils import parallel_processing + +nlp = spacy.load("en_core_web_sm") +logger = logging.getLogger(__name__) + + +def swap_and_filter_dict(dct: dict[str, Any], filter_to_keys: set[str]): + new_dct: dict[str, set[str]] = {} + for key, val in dct.items(): + if val in new_dct: + new_dct[val].add(key) + else: + new_dct[val] = {key} + + return {key: frozenset(val) for key, val in new_dct.items() if key in filter_to_keys} + + +def fill_reference_segments(record: ReferenceRecord, n_sent_before: int = 1, n_sent_after: int = 0) -> ReferenceRecord: + """ + Compute indices of the sentences containing the reference keyword, take their surrounding sentences and join them. + """ + + def compute_surroundings(hit_index: int, max_index: int, n_before: int, n_after: int): + """ + Computes indices of sentences to join into a coherent paragraph based on their location in text. + Ideally we would like to take (hit_index - n_before, hit_index + n_after), but we need to make sure + that we do not go out of bounds. + """ + lower = max(0, hit_index - n_before) + upper = min(max_index, hit_index + n_after) + return range(lower, upper + 1) + + with record.processed_data_source_path.open("r") as handle: + data = handle.read() + + sents = [sent.text for sent in nlp(data).sents] + indices_of_relevant_sents = [sents.index(x) for x in sents if any(y in x for y in record.actual_reference_keywords)] + + if not indices_of_relevant_sents: + record.segments = None + return record + + sequences_to_take = [ + compute_surroundings(x, len(sents) - 1, n_sent_before, n_sent_after) for x in indices_of_relevant_sents + ] + record.segments = {"".join([sents[y] for y in x]) for x in sequences_to_take} + + return record + + +def preprocess_data_source(record: ReferenceRecord) -> ReferenceRecord: + # TODO: There's some space for improvement, the preprocessing is acutally run twice. + + with record.raw_data_source_path.open("r") as handle: + data = handle.read() + + processed_data = preprocess_txt_func(data, record.actual_reference_keywords) + + with record.processed_data_source_path.open("w") as handle: + handle.write(processed_data) + + return record + + +def find_bracket_pattern(sentences: set[str], actual_reference_keywords: frozenset[str]): + patterns = [r"(\[.+?\])(?=.*" + x + r")" for x in actual_reference_keywords] + res: list[tuple[str, str]] = [] + + for sent in sentences: + for pattern, keyword in zip(patterns, actual_reference_keywords): + matches = re.findall(pattern, sent, flags=re.MULTILINE | re.UNICODE | re.DOTALL) + if matches: + res.append((matches[-1], keyword)) + return res + + +def preprocess_txt_func(data: str, actual_reference_keywords: frozenset[str]) -> str: + data = replace_acronyms(data) + data = replace_citation_identifiers(data, actual_reference_keywords) + return data + + +def replace_citation_identifiers(data: str, actual_reference_keywords: frozenset[str]) -> str: + segments = {sent.text for sent in nlp(data).sents if any(x in sent.text for x in actual_reference_keywords)} + patterns_to_replace = find_bracket_pattern(segments, actual_reference_keywords) + for x in patterns_to_replace: + data = data.replace(x[0], x[1]) + return data + + +def replace_acronyms(text: str) -> str: + acronym_replacements = { + "TOE": "target of evaluation", + "CC": "certification framework", + "PP": "protection profile", + "ST": "security target", + "SFR": "security Functional Requirement", + "SFRs": "security Functional Requirements", + "IC": "integrated circuit", + "MRTD": "machine readable travel document", + "TSF": "security functions of target of evaluation", + "PACE": "password authenticated connection establishment", + } + + for acronym, replacement in acronym_replacements.items(): + pattern = rf"(? tuple[str, str, frozenset[str], str, set[str] | None]: + return ( + self.certificate_dgst, + self.canonical_reference_keyword, + self.actual_reference_keywords, + self.source, + self.segments, + ) + + +class ReferenceSegmentExtractor: + """ + Class to process list of certificates into a dataframe that holds reference segments. + Should be only called with ReferenceSegmentExtractor()(list_of_certificates) + """ + + def __init__(self, n_sents_before: int = 1, n_sents_after: int = 0): + self.n_sents_before = n_sents_before + self.n_sents_after = n_sents_after + + def __call__(self, certs: Iterable[CCCertificate]) -> pd.DataFrame: + return self._prepare_df_from_cc_dset(certs) + + def _prepare_df_from_cc_dset(self, certs: Iterable[CCCertificate]) -> pd.DataFrame: + """ + Prepares processed DataFrame for reference annotator training from a list of certificates. This method: + - Extracts text segments relevant for each reference out of the certificates, forms dataframe from those + - Loads data splits into train/valid/test (unseen certificates are put into test set) + - Loads manually annotated samples + - Combines all of that into single dataframe + """ + target_certs = [x for x in certs if x.heuristics.st_references.directly_referencing and x.state.st_txt_path] + report_certs = [ + x for x in certs if x.heuristics.report_references.directly_referencing and x.state.report_txt_path + ] + df_targets = self._build_df(target_certs, "target") + df_reports = self._build_df(report_certs, "report") + print(f"df_targets shape: {df_targets.shape}") + print(f"df_reports shape: {df_reports.shape}") + + return ReferenceSegmentExtractor._process_df(pd.concat([df_targets, df_reports]), certs) + + def _build_records(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> list[ReferenceRecord]: + def get_cert_records(cert: CCCertificate, source: Literal["target", "report"]) -> list[ReferenceRecord]: + canonical_ref_var = {"target": "st_references", "report": "report_references"} + actual_ref_var = {"target": "st_keywords", "report": "report_keywords"} + raw_source_var = {"target": "st_txt_path", "report": "report_txt_path"} + + canonical_references = getattr(cert.heuristics, canonical_ref_var[source]).directly_referencing + actual_references = getattr(cert.pdf_data, actual_ref_var[source])["cc_cert_id"] + actual_references = { + inner_key: CertificateId(outer_key, inner_key).canonical + for outer_key, val in actual_references.items() + for inner_key in val + } + actual_references = swap_and_filter_dict(actual_references, canonical_references) + + raw_source_dir = getattr(cert.state, raw_source_var[source]).parent + processed_source_dir = raw_source_dir.parent / "txt_processed" + + return [ + ReferenceRecord( + cert.dgst, + raw_source_dir / f"{cert.dgst}.txt", + processed_source_dir / f"{cert.dgst}.txt", + key, + val, + source, + ) + for key, val in actual_references.items() + ] + + (certs[0].state.report_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True) + (certs[0].state.st_txt_path.parent.parent / "txt_processed").mkdir(exist_ok=True, parents=True) + return list(itertools.chain.from_iterable(get_cert_records(cert, source) for cert in certs)) + + def _build_df(self, certs: list[CCCertificate], source: Literal["target", "report"]) -> pd.DataFrame: + records = self._build_records(certs, source) + + records = parallel_processing.process_parallel( + preprocess_data_source, + records, + use_threading=False, + progress_bar=True, + progress_bar_desc="Preprocessing data", + ) + records_with_args = [(x, self.n_sents_before, self.n_sents_after) for x in records] + + results = parallel_processing.process_parallel( + fill_reference_segments, + records_with_args, + unpack=True, + use_threading=False, + progress_bar=True, + progress_bar_desc="Recovering reference segments", + ) + + print(f"I now have {len(results)} in {source} mode") + return pd.DataFrame.from_records( + [x.to_pandas_tuple() for x in results], + columns=["dgst", "canonical_reference_keyword", "actual_reference_keywords", "source", "segments"], + ) + + @staticmethod + def _get_split_dict() -> dict[str, str]: + """ + Returns dictionary that maps dgst: split, where split in `train`, `valid`, `test` + """ + + def get_single_dct(pth: Path, split_name: str) -> dict[str, str]: + with pth.open("r") as handle: + return dict.fromkeys(json.load(handle), split_name) + + split_directory = Path(str(files("sec_certs.data") / "reference_annotations/split/")) + return { + **get_single_dct(split_directory / "train.json", "train"), + **get_single_dct(split_directory / "valid.json", "valid"), + **get_single_dct(split_directory / "test.json", "test"), + } + + @staticmethod + def _get_annotations_dict() -> dict[tuple[str, str], str]: + """ + Returns dictionary mapping tuples `(dgst, canonical_reference_keyword) -> label` + """ + + def load_single_df(pth: Path, split_name: str) -> pd.DataFrame: + return ( + pd.read_csv( + pth, + na_values=["None"], + dtype={ + "dgst": str, + "canonical_reference_keyword": str, + "source": str, + "label": str, + "comment": str, + }, + ) + .dropna(subset="label") + .assign(label=lambda df_: df_.label.str.replace(" ", "_").str.upper(), split=split_name) + ) + + annotations_directory = Path(str(files("sec_certs.data") / "reference_annotations/final/")) + df_annot = pd.concat( + [ + load_single_df(annotations_directory / "train.csv", "train"), + load_single_df(annotations_directory / "valid.csv", "valid"), + load_single_df(annotations_directory / "test.csv", "test"), + ] + ) + + return ( + df_annot[["dgst", "canonical_reference_keyword", "label"]] + .set_index(["dgst", "canonical_reference_keyword"]) + .label.to_dict() + ) + + @staticmethod + def _process_df(df: pd.DataFrame, certs: Iterable[CCCertificate]) -> pd.DataFrame: + def process_segment(segment: str, actual_reference_keywords: frozenset[str]) -> str: + segment = " ".join(segment.split()) + for ref_id in actual_reference_keywords: + segment = segment.replace(ref_id, "REFERENCED_CERTIFICATE_ID") + return segment + + def unique_elements(series): + combined = [item for sublist in series for item in sublist] + return list(set(combined)) + + """ + Fully processes the dataframe. + """ + annotations_dict = ReferenceSegmentExtractor._get_annotations_dict() + split_dct = ReferenceSegmentExtractor._get_split_dict() + logger.info(f"Deleting {df.loc[df.segments.isnull()].shape[0]} rows with no segments.") + + df_new = df.copy() + df_new["full_key"] = df_new.apply(lambda x: (x["dgst"], x["canonical_reference_keyword"]), axis=1) + to_delete = len(df_new.loc[df_new.segments.isnull()].full_key.unique()) + print( + f"Deleting records for {to_delete} unique (dgst, referenced_id) pairs, not necessarily labeled ones. These have empty segments." + ) + + df_processed = ( + df.loc[df.segments.notnull()] + .explode("segments") + # .assign(lang=lambda df_: df_.segments.map(langdetect.detect)) + # .loc[lambda df_: df_.lang.isin({"en", "fr", "de"})] # This could get disabled possibly. + .groupby( + ["dgst", "canonical_reference_keyword"], + as_index=False, + dropna=False, + ) + .agg({"segments": list, "actual_reference_keywords": unique_elements}) + .assign( + actual_reference_keywords=lambda df_: df_.actual_reference_keywords.map(list), + label=lambda df_: [ + annotations_dict.get(x) for x in zip(df_["dgst"], df_["canonical_reference_keyword"]) + ], + split=lambda df_: df_.dgst.map(split_dct), + ) + .assign( + label=lambda df_: df_.label.map(lambda x: x if x is not None else np.nan), + split=lambda df_: df_.split.map(lambda x: "test" if pd.isnull(x) else x), + ) + ) + df_processed.segments = df_processed.apply( + lambda row: [process_segment(x, row.actual_reference_keywords) for x in row.segments], axis=1 + ) + return df_processed diff --git a/src/sec_certs/model/references_nlp/training.py b/src/sec_certs/model/references_nlp/training.py new file mode 100644 index 00000000..4e0f82ce --- /dev/null +++ b/src/sec_certs/model/references_nlp/training.py @@ -0,0 +1,68 @@ +import logging +import os + +import numpy as np +import pandas as pd +from catboost import CatBoostClassifier, Pool +from sklearn.dummy import DummyClassifier +from sklearn.metrics import balanced_accuracy_score +from sklearn.model_selection import KFold + +from sec_certs.constants import RANDOM_STATE, REF_ANNOTATION_MODES +from sec_certs.model.references_nlp.feature_extraction import get_data_for_clf + +logger = logging.getLogger(__name__) + + +def _train_model(x_train, y_train, x_eval, y_eval, learning_rate: float = 0.03, depth: int = 6, l2_leaf_reg: int = 3): + clf = CatBoostClassifier( + learning_rate=learning_rate, + depth=depth, + l2_leaf_reg=l2_leaf_reg, + task_type="GPU", + devices=os.environ["CUDA_VISIBLE_DEVICES"], + random_seed=RANDOM_STATE, + ) + + train_pool = Pool(x_train, y_train) + eval_pool = Pool(x_eval, y_eval) + clf.fit(train_pool, eval_set=eval_pool, verbose=False, plot=True, early_stopping_rounds=100, use_best_model=True) + return clf + + +def train_model( + df: pd.DataFrame, + mode: REF_ANNOTATION_MODES, + train_baseline: bool = False, + use_pca: bool = True, + use_umap: bool = True, + use_lang: bool = True, + use_pred: bool = True, + learning_rate: float = 0.03, + depth: int = 6, + l2_leaf_reg: int = 3, +) -> tuple[DummyClassifier | CatBoostClassifier, pd.DataFrame, list[str]]: + logger.info(f"Training model for mode {mode}") + X_train, y_train, eval_df, feature_cols = get_data_for_clf(df, mode, use_pca, use_umap, use_lang, use_pred) + if train_baseline: + clf = DummyClassifier(random_state=RANDOM_STATE) + clf.fit(X_train, y_train) + else: + assert eval_df is not None + clf = _train_model(X_train, y_train, eval_df[feature_cols], eval_df.label, learning_rate, depth, l2_leaf_reg) + + return clf, eval_df, feature_cols + + +def cross_validate_model(df: pd.DataFrame, learning_rate: float = 0.03, depth: int = 6, l2_leaf_reg: int = 3) -> float: + logger.info("Cross-validating model") + X_train, y_train, _, _ = get_data_for_clf(df, "cross-validation", True, True, True, True) + kf = KFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE) + scores = [] + for train_index, test_index in kf.split(X_train): + X_train_, X_test_ = X_train[train_index], X_train[test_index] + y_train_, y_test_ = y_train[train_index], y_train[test_index] + clf = _train_model(X_train_, y_train_, X_test_, y_test_, learning_rate, depth, l2_leaf_reg) + scores.append(balanced_accuracy_score(y_test_, clf.predict(X_test_))) + + return np.mean(scores) -- cgit v1.3.1