Final evaluation changes.

6 files changed, 277 insertions, 339 deletions

diff --git a/data/cert_id_eval/duplicate_ids.csv b/data/cert_id_eval/duplicate_ids.csv
index a29eabec..4e08e6a8 100644
--- a/data/cert_id_eval/duplicate_ids.csv
+++ b/data/cert_id_eval/duplicate_ids.csv
@@ -1,67 +1,13 @@
 cert_id,id,result,true_id,reason,fixable
-ANSSI-CC-2012/70,1b20b1629ba7d3a6,fp,KECS-ISIS-0490-2014,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2012/70,cb203cf5d91b1ae3,fp,KECS-ISIS-0533-2014,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2012/70,8870ddf70bf57890,fp,KECS-ISIS-0513-2014,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2012/70,3ea02a62856c752a,fp,KECS-ISIS-0468-2013,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2012/70,259e0ab1c4b690cf,fp,KECS-ISIS-0456-2013,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2012/70,aabd0ff0d82fff72,tp,ANSSI-CC-2012/70,,
-ANSSI-CC-2012/70,bcac7e3f9f614ec6,fp,KECS-ISIS-0675-2015,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2012/70,f2327302df1aa660,fp,KECS-ISIS-0435-2013,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0645-2010,a60b819509d62b84,tp,BSI-DSZ-CC-0645-2010,,
-BSI-DSZ-CC-0645-2010,720aab4f782f4c60,fp,2011-14-INF-1095,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0645-2010,9e7c513fdd35df17,fp,2011-13-INF-1092,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0645-2010,2fb4fdf2a570c366,fp,2011-15-INF-1098,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0645-2010,a5e8267322d0d7a6,fp,2011-12-INF-1089,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-1110-V3-2020,5346d5a4418466ab,fp,KECS-ISIS-1031-2020,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-1110-V3-2020,a7840a951b1de6c2,fp,ANSSI-CC-2020/87,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-1110-V3-2020,09220515a7edf3f9,tp,BSI-DSZ-CC-1110-V3-2020,,
-BSI-DSZ-CC-1110-V3-2020,8b7145c13f36c324,fp,ANSSI-CC-2020/88,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0782-V5-2020,15d68159595eae09,tp,BSI-DSZ-CC-0782-V5-2020,,
-BSI-DSZ-CC-0782-V5-2020,c8ff17063166050d,fp,KECS-ISIS-1140-2021,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0782-V5-2020,f2e7c6dfd6a431c8,fp,KECS-ISIS-1139-2021,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0782-V5-2020,17a23970b35a4f44,fp,KECS-ISIS-1113-2021,number of mentions of component certificate higher than actual,scheme mismatch
 SERTIT-014,a5fd51390f814a46,tp,SERTIT-014,,
 SERTIT-014,8278a9ba639a08d0,fp,,wrong certification report uploaded + clone,
 SERTIT-014,3ba740ca6ee8ed75,fp,,wrong certification report uploaded + clone,
-ANSSI-CC-2021/50,425f13fb7e94add5,tp,ANSSI-CC-2021/50,,
-ANSSI-CC-2021/50,b42009eb34746731,fp,ANSSI-CC-2022/29,number of mentions of component certificate higher than actual,frontpage match + file metadata
-ANSSI-CC-2021/50,0d2cab8061464438,fp,ANSSI-CC-2022/25,number of mentions of component certificate higher than actual,frontpage match + file metadata
-ANSSI-CC-2014/58,27d11629261d8806,fp,KECS-ISIS-0798-2017,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2014/58,627bfd69c2a831cf,tp,ANSSI-CC-2014/58,,
-ANSSI-CC-2014/58,cacdfdafce47678c,fp,KECS-ISIS-0799-2017,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0223-2003,4408217c4ce524fd,fp,ANSSI-CC-2003/12,number of mentions of component certificate higher than actual + clone,scheme mismatch
-BSI-DSZ-CC-0223-2003,b82b48b0e915aaac,tp,BSI-DSZ-CC-0223-2003,,
-BSI-DSZ-CC-0223-2003,c3f8e8ddb1dabb02,fp,ANSSI-CC-2003/12,number of mentions of component certificate higher than actual + clone,scheme mismatch
-ANSSI-CC-2019/01,f4b4b026e87b5086,fp,KECS-ISIS-0937-2019,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2019/01,fbc78e2407910a4b,fp,KECS-ISIS-0936-2019,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2019/01,84c8bf1f809380bc,tp,ANSSI-CC-2019/01,,
-ANSSI-CC-2018/12,a0c38b4389ad7cf7,fp,2017-63-INF-3232,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2018/12,10107b83e7393c50,tp,ANSSI-CC-2018/12,,
-ANSSI-CC-2018/12,7dfcefd1ffeaa941,fp,2017-62-INF-3233,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2021/49,6fd73557577cf9ed,tp,ANSSI-CC-2021/49,,
-ANSSI-CC-2021/49,037577fc2019fcfa,fp,ANSSI-CC-2022/28,number of mentions of component certificate higher than actual,frontpage match + file metadata
-ANSSI-CC-2021/49,d55eb8c97d71e843,fp,ANSSI-CC-2022/24,number of mentions of component certificate higher than actual,frontpage match + file metadata
 ANSSI-CC-2009/36,fad651102525c5a4,fp,ANSSI-CC-2009/48,mismatched frontpage and title page + clone,
 ANSSI-CC-2009/36,928e364f71d64595,fp,ANSSI-CC-2009/48,mismatched frontpage and title page + clone,
 ANSSI-CC-2009/36,691b730a7bbb28b0,tp,ANSSI-CC-2009/36,,
-567-LSS,0a06ae047b9790ba,tp,567-LSS,clone + removed,handle removed
-567-LSS,53231c8ddc542543,tp,567-LSS,clone,
-567-LSS,3af72451867b3187,tp,567-LSS,clone + removed,handle removed
 CCEVS-VR-VID11024-2019,dfa12d7893c9efdf,fp,,wrong certification report and security target uploaded,
 CCEVS-VR-VID11024-2019,20f851314c4f57d9,tp,CCEVS-VR-VID11024-2019,,
 CCEVS-VR-VID11024-2019,b4288d36bc129d2f,fp,,wrong certification report and security target uploaded,
-BSI-DSZ-CC-0782-V2-2015,188d3cda4e6f2d5d,fp,KECS-ISIS-0677-2015,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0782-V2-2015,2a91f42389fda90d,fp,KECS-ISIS-0676-2015,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0782-V2-2015,446aa68e0c4c5083,tp,BSI-DSZ-CC-0782-V2-2015,,
-ANSSI-CC-2021/52,d355c778fcfee3a9,fp,ANSSI-CC-2022/27,number of mentions of component certificate higher than actual,frontpage match + file metadata
-ANSSI-CC-2021/52,6cef8442092da1da,fp,ANSSI-CC-2022/31,number of mentions of component certificate higher than actual,frontpage match + file metadata
-ANSSI-CC-2021/52,536aac44b608f951,tp,ANSSI-CC-2021/52,,
-ANSSI-CC-2021/51,a47713a44e4c656b,tp,ANSSI-CC-2021/51,,
-ANSSI-CC-2021/51,940a18f62c79ffae,fp,ANSSI-CC-2022/30,number of mentions of component certificate higher than actual,frontpage match + file metadata
-ANSSI-CC-2021/51,5787ff8207fd751d,fp,ANSSI-CC-2022/26,number of mentions of component certificate higher than actual,frontpage match + file metadata
-BSI-DSZ-CC-0945-2017,457fcec1ad31841f,fp,ANSSI-CC-2022/37v2,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0945-2017,0b36924a005c2ae2,fp,ANSSI-CC-2022/36v2,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0945-2017,5103648897ba1b3a,tp,BSI-DSZ-CC-0945-2017,,
 CCEVS-VR-05-0091,c79dafc7b0da855e,tp,CCEVS-VR-05-0091,clone,
 CCEVS-VR-05-0091,355a5b9240e32cc9,tp,CCEVS-VR-05-0091,clone,
 2009-27-INF-754,7ac69e82d1a680bb,fp,,different reports share cert_id,
@@ -70,16 +16,12 @@ ANSSI-CC-2021/16,0ef56cba2362e463,tp,ANSSI-CC-2021/16,,
 ANSSI-CC-2021/16,875347e0d906197d,fp,ANSSI-CC-2021/16-R01,regex issue,regex fix
 ANSSI-CC-2011/72,2b0342d5b70b7a78,fp,ANSSI-CC-2011/82,mismatched frontpage and title page,
 ANSSI-CC-2011/72,dec61898ab005e45,tp,ANSSI-CC-2011/72,,
-ANSSI-CC-2021/34,60f31793e1e5f4e1,tp,ANSSI-CC-2021/34,,
-ANSSI-CC-2021/34,3cbdd57de3f32ad0,fp,ANSSI-CC-2021/34-R01,regex issue,regex fix
 CCEVS-VR-10786-2017,d976861ecbeee69c,fp,,different reports share cert_id,
 CCEVS-VR-10786-2017,371abb3484840317,fp,,different reports share cert_id,
 ANSSI-CC-2020/65,d492f0e3e19d32f6,fp,ANSSI-CC-2020/65v2,regex substring,frontpage match
 ANSSI-CC-2020/65,b30ef2c1aaec9236,tp,ANSSI-CC-2020/65,,
 BSI-DSZ-CC-0588-2009,03b4ff588fb88dae,fp,,wrong certification report ,
 BSI-DSZ-CC-0588-2009,031667f4e242da61,tp,BSI-DSZ-CC-0588-2009,,
-ANSSI-CC-2021/62,559e4549a852aab2,tp,ANSSI-CC-2021/62,,
-ANSSI-CC-2021/62,3e020408537aaad0,fp,ANSSI-CC-2021/62-R01,regex issue,regex fix
 ANSSI-CC-2020/12,6c6c5ba82977609b,fp,ANSSI-CC-2020/93,number of mentions of component certificate higher than actual,frontpage match
 ANSSI-CC-2020/12,af6767c887ae930a,tp,ANSSI-CC-2020/12,,
 BSI-DSZ-CC-1139-V3-2021,887139711564fdb3,tp,BSI-DSZ-CC-1139-V3-2021,clone,
@@ -104,36 +46,20 @@ CCEVS-VR-VID10510-2014,965f41d028746f9b,fp,,different reports share cert_id,
 CCEVS-VR-VID10510-2014,0cf98690ba05ef68,fp,,different reports share cert_id,
 383-4-263-CR,f2032ab2a50ab35d,tp,383-4-263,clone,
 383-4-263-CR,1fb95703e4e12a49,tp,383-4-263,clone,
-NSCIB-CC-0237694-CR,9fd2cc58ecfab9c4,tp,NSCIB-CC-0237694-CR,clone,
-NSCIB-CC-0237694-CR,b1401488f998c1f0,tp,NSCIB-CC-0237694-CR,clone,
-ANSSI-CC-2020/07,8820852f4a1163b3,tp,ANSSI-CC-2020/07,clone,
-ANSSI-CC-2020/07,9ab449ecd0da1754,tp,ANSSI-CC-2020/07,clone,
 CCEVS-VR-05-0124,8bf284d3b482ad65,tp,CCEVS-VR-05-0124,clone,
 CCEVS-VR-05-0124,3ad601a9cb30dfef,tp,CCEVS-VR-05-0124,clone,
 CCEVS-VR-05-0102,a67fe4216fd88917,tp,CCEVS-VR-05-0102,clone,
 CCEVS-VR-05-0102,ec25be3458d728cd,tp,CCEVS-VR-05-0102,clone,
 383-4-198-CR,3c24bbe7724dbfdd,tp,383-4-198,clone,
 383-4-198-CR,0a3793d207c47790,tp,383-4-198,clone,
-NSCIB-CC-0308282-CR,01fa8bfbc95c23b5,tp,NSCIB-CC-0308282-CR,clone,
-NSCIB-CC-0308282-CR,f6e744b5e5ae6d4f,tp,NSCIB-CC-0308282-CR,clone + removed,handle removed
-ANSSI-CC-2014/59,ac4687f4b15507b6,fp,KECS-ISIS-0797-2017,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2014/59,6d41460480570d22,tp,ANSSI-CC-2014/59,,
 CCEVS-VR-VID10327-2009,c80e1ca4d8641a3d,tp,CCEVS-VR-VID10327-2009,clone,
 CCEVS-VR-VID10327-2009,0f6c8bde7c6574e3,tp,CCEVS-VR-VID10327-2009,clone,
-NSCIB-CC-0388535-CR,3a89942886178cad,tp,NSCIB-CC-0388535-CR,clone + removed,handle removed
-NSCIB-CC-0388535-CR,ab4814d2b8c742a0,tp,NSCIB-CC-0388535-CR,clone,
 ANSSI-CC-2010/36,5e4c757231135ba1,tp,ANSSI-CC-2010/36,clone,
 ANSSI-CC-2010/36,69c75283a4ccf934,tp,ANSSI-CC-2010/36,clone,
-BSI-DSZ-CC-0950-V2-2018,a46522d0a592c1fe,tp,BSI-DSZ-CC-0950-V2-2018,clone + removed,handle removed
-BSI-DSZ-CC-0950-V2-2018,6f90b0bf6bbe884b,tp,BSI-DSZ-CC-0950-V2-2018,clone,
-Rapport de certification 2005/01,f3bb1641b4bb0aa0,tp,ANSSI-CC-2005/01,clone,
-Rapport de certification 2005/01,749fae44017016c5,tp,ANSSI-CC-2005/01,clone + mismatched scheme,scheme mismatch
 2004-3-INF-71,1bb3da30bc8f4750,tp,2004-3-INF-71,clone,
 2004-3-INF-71,0b0648a9b9a26747,tp,2004-3-INF-71,clone,
 CCEVS-VR-VID10559-2014,aa49720eb88f291c,tp,CCEVS-VR-VID10559-2014,,
 CCEVS-VR-VID10559-2014,449ea70a6342d680,fp,,wrong certification report,
-ANSSI-CC-2021/30,c62b878b1ac43df9,tp,ANSSI-CC-2021/30,,
-ANSSI-CC-2021/30,459539da9315ccac,fp,ANSSI-CC-2021/30-R01,regex issue,regex fix
 ANSSI-CC-2010/24,0372b733c28f305a,tp,ANSSI-CC-2010/24,clone,
 ANSSI-CC-2010/24,b9789028f97490ce,tp,ANSSI-CC-2010/24,clone,
 ANSSI-CC-2010/34,2f53e1d94ca1a010,tp,ANSSI-CC-2010/34,clone,
@@ -148,20 +74,12 @@ ANSSI-CC-2014/36,8d6bca0d7f228191,tp,ANSSI-CC-2014/36,clone,
 ANSSI-CC-2014/36,f43765c81befb944,tp,ANSSI-CC-2014/36,clone,
 CCEVS-VR-VID10003-2008,bf7b3c78598f74cd,tp,CCEVS-VR-VID10003-2008,,
 CCEVS-VR-VID10003-2008,8f538d2eb9a1c6c7,fp,,wrong certification report,
-BSI-DSZ-CC-0891-V2-2016,bc715a45d7584d95,fp,NSCIB-CC-16-95781-CR,number of mentions of component certificate higher than actual,frontpage match + scheme mismatch
-BSI-DSZ-CC-0891-V2-2016,2d835bff00514238,tp,BSI-DSZ-CC-0891-V2-2016,,
-BSI-DSZ-CC-0935-2015,025ccea9d90154b0,tp,BSI-DSZ-CC-0935-2015,,
-BSI-DSZ-CC-0935-2015,a4bbf07046109fc9,fp,NSCIB-CC-10-30076-CR,number of mentions of component certificate higher than actual,scheme mismatch
 ANSSI-CC-2009/46,baead73ecc6e4a33,tp,ANSSI-CC-2009/46,clone,
 ANSSI-CC-2009/46,d76f210e41d5589c,tp,ANSSI-CC-2009/46,clone,
-BSI-DSZ-CC-0234-2004,f577607298c4b574,fp,CRP212,number of mentions of component certificate higher than actual,scheme mismatch
-BSI-DSZ-CC-0234-2004,1cb63a13bf4c58ff,tp,BSI-DSZ-CC-0234-2004,,
 BSI-DSZ-CC-1053-2018,4e246ff7fd346630,tp,BSI-DSZ-CC-1053-2018,clone,
 BSI-DSZ-CC-1053-2018,a6629b6ce5514a43,tp,BSI-DSZ-CC-1053-2018,clone,
 ANSSI-CC-2018/41,10b17081dd7cad8f,tp,ANSSI-CC-2018/41,clone,
 ANSSI-CC-2018/41,27baf39406deb767,tp,ANSSI-CC-2018/41,clone,
-ANSSI-CC-2021/02,82c717a9a379b75c,tp,ANSSI-CC-2021/02,,
-ANSSI-CC-2021/02,7e24168630d5124e,fp,ANSSI-CC-2021/02-R01,regex issue,regex fix
 ANSSI-CC-2009/49,1b5a04adf5047f67,tp,ANSSI-CC-2009/49,clone,
 ANSSI-CC-2009/49,1188aebd8681e4fe,tp,ANSSI-CC-2009/49,clone,
 CCEVS-VR-05-0137,feeecec3ac6f0503,tp,CCEVS-VR-05-0137,clone,
@@ -170,26 +88,14 @@ ANSSI-CC-2013/09,a5adb726852f5cc5,tp,ANSSI-CC-2013/09,clone,
 ANSSI-CC-2013/09,ee35dcafbe1b9532,tp,ANSSI-CC-2013/09,clone,
 ANSSI-CC-2010/03,67a79bfc84b4071b,tp,ANSSI-CC-2010/03,clone,
 ANSSI-CC-2010/03,8826a8ae0369901a,tp,ANSSI-CC-2010/03,clone,
-NSCIB-CC-98209-CR5,af86594cd7aaf6d4,tp,NSCIB-CC-98209-CR5,clone + removed,handle removed
-NSCIB-CC-98209-CR5,6fc6a0a8163fea0c,tp,NSCIB-CC-98209-CR5,clone,
-BSI-DSZ-CC-0501-2008,78bb780793fbb176,tp,BSI-DSZ-CC-0501-2008,,
-BSI-DSZ-CC-0501-2008,f7cbb33cc639b5b5,fp,KECS-ISIS-0222-2010,number of mentions of component certificate higher than actual,scheme mismatch
 BSI-DSZ-CC-0196-2003,063d4ed52b084441,fp,,wrong certification report ,
 BSI-DSZ-CC-0196-2003,41708ba21f2bb4a7,tp,BSI-DSZ-CC-0196-2003,,
-BSI-DSZ-CC-0624-2010,09be49631855ae19,tp,BSI-DSZ-CC-0624-2010,,
-BSI-DSZ-CC-0624-2010,1816ddecd7432cf9,fp,KECS-ISIS-0378-2012,number of mentions of component certificate higher than actual,
 383-4-100-CR,8ba5d4d02ea73d3d,tp,383-4-100,,scheme mismatch
 383-4-100-CR,c4e1391e8ba34694,fp,,wrong certification report ,
-NSCIB-CC-0441513-CR,7533a87b0379ce3e,tp,NSCIB-CC-0441513-CR,clone + removed,handle removed
-NSCIB-CC-0441513-CR,82c6207791448e0c,tp,NSCIB-CC-0441513-CR,clone,
 ANSSI-CC-2019/20,c36783a79de002c5,tp,ANSSI-CC-2019/20,clone,
 ANSSI-CC-2019/20,088504703b9eedc1,tp,ANSSI-CC-2019/20,clone,
-CSEC2020024,f2b4285a59a296f2,tp,CSEC2020024,clone + removed,handle removed
-CSEC2020024,e502549eb7fd9869,tp,CSEC2020024,clone,
 CCEVS-VR-10610-2015,6f0a50d3bd7db7c7,fp,,wrong certification report ,
 CCEVS-VR-10610-2015,e93207ee36a5fabf,tp,CCEVS-VR-10610-2015,,
-NSCIB-CC-0237686-CR,294ef6cb28408ec9,tp,NSCIB-CC-0237686-CR,clone,
-NSCIB-CC-0237686-CR,6bca140482214182,tp,NSCIB-CC-0237686-CR,clone + removed,handle removed
 CCEVS-VR-04-0074,d42974d5cce56320,tp,CCEVS-VR-04-0074,clone,
 CCEVS-VR-04-0074,1bcac3b368582ae1,tp,CCEVS-VR-04-0074,clone,
 ANSSI-CC-2021/37,5498bfcecf19dfa3,tp,ANSSI-CC-2021/37,,
@@ -198,31 +104,37 @@ CCEVS-VR-06-0024,686005d0b5ff5c5c,tp,CCEVS-VR-06-0024,clone,
 CCEVS-VR-06-0024,b901bbd0ce1f0a3b,tp,CCEVS-VR-06-0024,clone,
 ANSSI-CC-2011/07,41bd2924e9afced5,tp,ANSSI-CC-2011/07,clone,
 ANSSI-CC-2011/07,796445972ef2ff5b,tp,ANSSI-CC-2011/07,clone,
-ANSSI-CC-2018/38,3459c14186e8223f,tp,ANSSI-CC-2018/38,,
-ANSSI-CC-2018/38,371dca18821f7714,fp,ANSSI-CC-2018/38v2,regex substring,frontpage match
-CRP292,a00e93f2981ef418,fp,CRP298,number of mentions of component certificate higher than actual,frontpage match
-CRP292,3e345d3c6002ad55,tp,CRP292,,
 ANSSI-2009/24,a52f807db0eb7fdb,tp,ANSSI-CC-2009/24,clone,
 ANSSI-2009/24,e6a8cbbc8c557bed,tp,ANSSI-CC-2009/24,clone,
-ANSSI-CC-2013/47,34370b67b5e675c3,fp,KECS-ISIS-0489-2014,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2013/47,6aa2f56da553d95d,tp,ANSSI-CC-2013/47,,
-ANSSI-CC-2010/02,7d4585a4b5b6e873,fp,KECS-ISIS-0419-2012,number of mentions of component certificate higher than actual,scheme mismatch
-ANSSI-CC-2010/02,dbbf02a1cd0ad33b,tp,ANSSI-CC-2010/02,,
 2016-47-INF-2126,f254bdb3e177193d,tp,2016-47-INF-2126,clone,
 2016-47-INF-2126,b1f92dfbc5d51547,tp,2016-47-INF-2126,clone,
 ANSSI-CC-2010/39,4e9bb649bce14c86,tp,ANSSI-CC-2010/39,clone,
 ANSSI-CC-2010/39,30ae700d801bd41c,tp,ANSSI-CC-2010/39,clone,
 ANSSI-CC-2014/46,da7fa774f888542b,tp,ANSSI-CC-2014/46,clone,
 ANSSI-CC-2014/46,ba5cbf3c600d99ac,tp,ANSSI-CC-2014/46,clone,
-ANSSI-CC-2021/35,14b5a3a574c90154,fp,ANSSI-CC-2021/35-R01,regex issue,regex fix
-ANSSI-CC-2021/35,ce22219c2049985c,tp,ANSSI-CC-2021/35,,
 CSEC2016007,13e306eb69c8bf95,tp,CSEC2016007,clone (recertification?),
 CSEC2016007,0c3ff6669dcf6968,tp,CSEC2016007,clone (recertification?),
-BSI-DSZ-CC-0639-2010,2f9feaf4121720da,tp,BSI-DSZ-CC-0639-2010,,
-BSI-DSZ-CC-0639-2010,db7627e56b8621b1,fp,KECS-ISIS-0394-2012,number of mentions of component certificate higher than actual,scheme mismatch
-Certification Report 98/94,28bb2a8869397c4c,fp,CRP106,number of mentions of component certificate higher than actual,scheme mismatch
-Certification Report 98/94,28a9fc2137d72ce9,fp,CRP103,number of mentions of component certificate higher than actual,scheme mismatch
 CSEC2015006,a5607124c92c10ed,tp,CSEC2015006,shared cert_id,
 CSEC2015006,f89c3a438204a45b,tp,CSEC2015006,shared cert_id,
-CRP208,b2aefe8b1bd4d9b3,fp,CRP208A,regex issue,regex fix
-CRP208,c90ddb9915291718,tp,CRP208,,
+ANSSI-CC-2009/47,80ae5f71b5fa278a,tp,ANSSI-CC-2009/47,clone,
+ANSSI-CC-2009/47,0cbcee9f1cde47a8,tp,ANSSI-CC-2009/47,clone,
+CRP141,e510cd16d4f9f4a3,tp,CRP141,clone,
+CRP141,b802ce16d43a545e,tp,CRP141,clone,
+ANSSI-CC-2009/25,fb40bf81f6774748,tp,ANSSI-CC-2009/25,,
+ANSSI-CC-2009/25,b54e179209c533c5,fp,ANSSI-CC-2009/57,mismatched frontpage and title page,
+ANSSI-CC-2018/57,cacd9550c69df042,tp,ANSSI-CC-2018/57,,
+ANSSI-CC-2018/57,449c74a92ebb61a4,fp,ANSSI-CC-2018/57v2,number of mentions of component certificate higher than actual,frontpage match
+CRP153,20c98a49ea86a519,tp,CRP153,clone,
+CRP153,a79f15874e255007,tp,CRP153,clone,
+ANSSI-CC-2018/32,82ade52e9ef12fd9,tp,ANSSI-CC-2018/32,,
+ANSSI-CC-2018/32,09b1192474bc8b27,fp,ANSSI-CC-2018/32v2,number of mentions of component certificate higher than actual,frontpage match
+383-4-335,3caad2930e3cbf2a,tp,383-4-335,,
+383-4-335,97a292f85b058e0d,fp,383-4-435,cert_id typo,
+KECS-NISS-0052-2006,59cbd48b9dc51e87,fp,KECS-NISS-0053-2006,filename typo,
+KECS-NISS-0052-2006,06f6d7907510c36e,tp,KECS-NISS-0052-2006,,
+ANSSI-CC-2018/58,744a7a202d909323,fp,ANSSI-CC-2018/58v2,number of mentions of component certificate higher than actual,frontpage match
+ANSSI-CC-2018/58,d57c84efd02c4bb7,tp,ANSSI-CC-2018/58,,
+CCEVS-VR-03-0044,814a2ac0aeae08b1,tp,CCEVS-VR-03-0044,clone,
+CCEVS-VR-03-0044,02a31d0ed42e4ff1,tp,CCEVS-VR-03-0044,clone,
+ANSSI-CC-2018/56,6df8f86449e56c05,tp,ANSSI-CC-2018/56,,
+ANSSI-CC-2018/56,c5e25f90c7006546,fp,ANSSI-CC-2018/56v2,number of mentions of component certificate higher than actual,frontpage match
\ No newline at end of file
diff --git a/data/cert_id_eval/missing_ids.csv b/data/cert_id_eval/missing_ids.csv
index 75da4baf..d9c38f5e 100644
--- a/data/cert_id_eval/missing_ids.csv
+++ b/data/cert_id_eval/missing_ids.csv
@@ -1,95 +1,56 @@
 id,scheme,cert_id,reason
-889693d2b11bdfaf,JP,C0023,not parsed due to PDF issue
-20c98a49ea86a519,UK,CRP153,
+889693d2b11bdfaf,JP,C0023,pdf issue
 a90d78924e91dc6c,FR,,404 for report PDF
-21d5f1b05e1ba491,JP,C0003,not parsed due to PDF issue
+21d5f1b05e1ba491,JP,C0003,pdf issue
 6af4a27251001715,US,,404 for report PDF
-3ad7749ef678ce95,JP,C0093,not parsed due to PDF issue
+3ad7749ef678ce95,JP,C0093,pdf issue
 d0d0dfe4860db6be,US,,404 for report PDF
 822e5f25e7c4631e,CA,,cert_id missing
-10dc636e9da3e1b6,CA,,cert_id missing
-357b38a376d955f7,JP,,has security target uploaded in place of report
-e2ecf0daf73716f2,CA,511-LSS,regex missing
+357b38a376d955f7,JP,C0678,has security target uploaded in place of report
 7d074a56ee50aec4,US,,404 for report PDF
 e27eb9efbd67c4f7,KR,,404 for report PDF
 811ca609aebe2e96,US,,404 for report PDF
-9219f2fb5f305269,KR,KECS-ISIS-0119-2008,has unicode hyphens 8208
-10339846e519597c,CA,516-LSS,regex missing
 4a1fa75170579066,US,,cert_id missing
-f0068ddc5569b772,CA,518-LSS,regex missing
-28fb117adbb809e3,SE,CSEC2006002,space in cert_id
-2d05e976e1adf696,CA,,report not a PDF (docx) also cert_id missing
-a79f15874e255007,UK,CRP153,
-7b1f816f20381626,US,,maybe "10-1904-R-0030 V1.1" the ETR id
-32d4bf275131aa15,KR,KECS-ISIS-0118-2008,has unicode hyphens 8208
+7b1f816f20381626,US,,cert_id missing
 f4dc9b09aa682008,US,,cert_id missing
 ed0b7793ac82822e,US,,cert_id missing
-bfc86f14e2484335,CA,,report not a PDF (docx) also cert_id missing
-1491021aa321b828,CA,517-CCS-2020,regex missing
-b762fcb2983aa148,JP,Certification No. C0046,not parsed due to PDF issue
+b762fcb2983aa148,JP,C0046,pdf issue
 e133881d7203a6e4,ES,INF-3516,regex missing
 305af0eec85bb856,KR,,404 for report PDF
-a91b4c64329d05a8,SE,CSEC2019010,space in cert_id
-c5bad5b309469e22,CA,519-EWA,regex missing
 08e892fc32b0579f,ES,INF-3483,regex missing
-876cb2fe02017f3f,JP,C0012,not parsed due to PDF issue
 0dcc691ea04d1b52,CA,383-4-351,certificate.docx
-f9c8da9deff77ab5,CA,,report not a PDF (docx) also cert_id missing
 ace7ea6d7f58dbb1,DE,,404 for report PDF
 7147c2f70d983d57,TR,21.0.03/TSE-CCCS-44,report not a PDF (docx)
-6266d99d3b983091,CA,,cert_id missing
-66838f2e4bb6eccc,FR,,has security target uploaded in place of report
-8ed77a6223a94fb5,US,,maybe "09-1832-R-0027 V1.0" the ETR id
+66838f2e4bb6eccc,FR,ANSSI-CC-2018/42,has security target uploaded in place of report
+8ed77a6223a94fb5,US,,cert_id missing
 422dc5758723c7d1,TR,21.0.03/TSE-CCCS-68,report not a PDF (docx)
 02482228eb547c15,US,,404 for report PDF
-4de4a9f436958574,CA,525-LSS-2020,report not a PDF (docx)
-c312bbf9e303b844,SE,CSEC2019012,space in cert_id
-fb6d7faf0ed2dca5,CA,,cert_id missing
-b3ae94eeccc566ba,CA,383-4-17,regex missing
 d3568613c552f9e8,KR,,malformed PDF
 b43100f226a61f53,FR,,malformed PDF
-0a117ebd82bdfb0c,US,,maybe "11-2280-R-0077 V1.1" the ETR id
+0a117ebd82bdfb0c,US,,cert_id missing
 95948dc5f3e472fb,IT,,cert_id missing
-9253ec0d31264194,US,,maybe "08-1602-R-0084 V1.1" the ETR id
-5f18321fb9a41409,KR,KECS-ISIS-0253-2010,has unicode hyphens 8208
+9253ec0d31264194,US,,cert_id missing
 1a8ac8fbe5bfcda9,NO,SERTIT-017,404 for report PDF
 3faae83d248da8ba,IT,,cert_id missing
-d5ed93f6586469f6,CA,503-LSS,regex missing
-a45f04b9a5244a51,KR,KECS-ISIS-0189-2009,has unicode hyphens 8208
-243a29b98f34ae9a,SE,CSEC2019009,space in cert_id
-88eb228eeb487597,KR,KECS-ISIS-0223-2010,has unicode hyphens 8208
-739f48568b127f22,CA,,cert_id missing
-71e471543ea7e03f,CA,383-4-56,regex missing
-ae1b80e9cccec491,JP,C0089,not parsed due to PDF issues
-5945cedd8cb01e06,JP,CRP-C0120-01,report and target documents switched
-7e664d089ed905a5,CA,383-4-422P,regex missing
+ae1b80e9cccec491,JP,C0089,pdf issue
+5945cedd8cb01e06,JP,C0120,report and target documents switched
 c508c550607efde5,US,,cert_id missing
-456bfb3665815ccb,KR,KECS-ISIS-0120-2008,has unicode hyphens 8208
-1683817e5db17d9c,JP,C0047,not parsed due to PDF issues
-b2aefe8b1bd4d9b3,UK,,cert_id missing
-f0af2383ad08e3ba,SE,CSEC2019011,regex missing
-ae78e8b68bd4c917,US,CCEVS-VR-CC-11074-2020,regex missing
+1683817e5db17d9c,JP,C0047,pdf issue
 323987cac96072cb,TR,14.10.01/10-105,cert_id malformed
 8ac22970bd8042d2,US,,404 for report PDF
-b61b6299f457a01e,SE,CSEC2006001,regex missing
-6fbb34a3a08e943c,CA,,cert_id missing
 6901124edcfe478f,US,CCEVS-VR-03-0038,report and target documents switched
 98668cafb8dda26f,SE,CSEC2019014,report not a PDF (docx)
 d79ecebfe21f2144,US,,404 for report PDF
 7e32023021d5aad2,ES,INF-510,regex missing
 02958808b57334f0,SE,CSEC2019016,report not a PDF (docx)
 391c8b9254ed6ed9,CA,,404 for report PDF
-d3b7b0df0108a466,CA,383-4-344,certificate.docx
 82fcce2db9ef6063,SG,,404 for report PDF
 5e646214ea5b1044,US,CCEVS-VR-VID11243-2022,has security target uploaded in place of report
-614f4dcdee0128bd,IN,IC3S/KOL01/ECITelecom/EAL2/0420/0019/CR,space in cert_id
-3477723044183b31,FR,ANSSI-CC-2017/42v2-S01,regex missing
-a5c2f0adcecf1a33,MY,ISCB-5-RPT-C127-CR-v1.1,regex missing
 d399d47f958a80dd,AU,Certificate Number: 2020/126,certificate.pdf + OCR
 8b357b0ccdafe13b,AU,Certificate Number: 2019/120,certificate.pdf + OCR
-4fb34c6529549e26,JP,C0064,?
+4fb34c6529549e26,JP,C0064,pdf issue
 c64192c1597a1a14,AU,,cert_id missing
-245f78711cfcb537,JP,C0142,?
+245f78711cfcb537,JP,C0142,pdf issue
 acd95d9dd4ad410c,AU,Certificate Number: 2019/121,certificate.pdf + OCR
 9903c89cbb35eec1,AU,Certificate Number: 2019/123,certificate.pdf + OCR
 908128ea95f94b09,AU,Certificate Number: 2020/132,certificate.pdf + OCR
@@ -98,22 +59,22 @@ acd95d9dd4ad410c,AU,Certificate Number: 2019/121,certificate.pdf + OCR
 5db22161b84ab3d5,AU,Certificate Number: 2020/127,certificate.pdf + OCR
 5390c9458174b933,AU,Certificate Number: 2020/129,certificate.pdf + OCR
 bd7d1fa4b1e938e4,AU,Certificate Number: 2020/128,certificate.pdf + OCR
-bd992ce6d76206bf,JP,C0203,?
-c74dd9d805e44354,FR,ANSSI-CC-2005/12,?
-ea708fac8f672c0f,JP,C0702,?
+bd992ce6d76206bf,JP,C0203,pdf issue
+c74dd9d805e44354,FR,ANSSI-CC-2005/12,pdf issue
+ea708fac8f672c0f,JP,C0702,pdf issue
 14e28608791e4a82,AU,,cert_id missing
 96cf0d5d7f98fbf1,AU,Certificate Number: 2020/130,certificate.pdf + OCR
 770f2b13b0723e49,AU,Certificate Number: 2021/134,certificate.pdf + OCR
-3891a6c29ed701b4,FR,ANSSI-CC-2005/02,?
+3891a6c29ed701b4,FR,ANSSI-CC-2005/02,pdf issue
 7a799fc315ee99da,AU,Certificate Number: 2021/133,certificate.pdf + OCR
 5f6946c674c32b30,AU,Certificate Number: 2022/138,certificate.pdf
-749fae44017016c5,FR,ANSSI-CC-2005/01,?
-3097b47b6f065ba5,JP,C0062,?
+749fae44017016c5,FR,ANSSI-CC-2005/01,pdf issue
+3097b47b6f065ba5,JP,C0062,pdf issue
 9a1c767d358eee50,AU,,cert_id missing
 5ceb859ce72daecf,AU,Certificate Number: 2021/135,certificate.pdf + OCR
 fdf20384e73efb57,AU,Certificate Number: 2020/131,certificate.pdf + OCR
 d2b49eb29a3dcbdc,US,,cert_id missing
 afcf5263d97159f8,AU,Certificate Number: 2019/125,certificate.pdf + OCR
-10d135ab24eab5a8,CA,383-4-270,?
-4408217c4ce524fd,FR,ANSSI-CC-2003/12,?
-c3f8e8ddb1dabb02,FR,ANSSI-CC-2003/12,?
\ No newline at end of file
+10d135ab24eab5a8,CA,383-4-270,pdf issue
+4408217c4ce524fd,FR,ANSSI-CC-2003/12,pdf issue
+c3f8e8ddb1dabb02,FR,ANSSI-CC-2003/12,pdf issue
\ No newline at end of file
diff --git a/data/cert_id_eval/random.csv b/data/cert_id_eval/random.csv
index 50522eb4..c995debd 100644
--- a/data/cert_id_eval/random.csv
+++ b/data/cert_id_eval/random.csv
@@ -1 +1,101 @@
-id,cert_id,source,possible
+id,cert_id
+61e7578e78fbe755,14.10.06/13-006
+dc91f94aa005a094,CCEVS-VR-VID10789-2017
+0349c893b81f7c1a,NSCIB-CC-0189590-CR
+ece463ca87392f7b,CCEVS-VR-06-0016
+c92a196d0408a5ff,ANSSI-CC-2013/57
+0e22fe4e4e58faf4,BSI-DSZ-CC-1052-V5-2022
+b4bec1e6f04c3533,BSI-DSZ-CC-0478-2008
+940a18f62c79ffae,ANSSI-CC-2022/30
+00fc58baa6297c0d,C0072
+29964f32c68b0ce8,BSI-DSZ-CC-0519-V4-2022
+b2f3874cdb97fe27,BSI-DSZ-CC-0790-2013
+1fb95703e4e12a49,383-4-263
+546f9e2496b1d11c,ANSSI-CC-2015/14
+1dd7f4bf2e677073,ANSSI-CC-2014/20
+b24a14935edd51ad,BSI-DSZ-CC-0783-2013
+302df01b5d463eea,CCEVS-VR-VID10253-2009
+6337d3dcc5362dc6,NSCIB-CC-0122197-CR
+8da378edb071a71c,CCEVS-VR-VID10691-2016
+dd1482a4911a3f1d,CCEVS-VR-05-0105
+99223aca5d9eb3b3,ANSSI-CC-2011/01
+dc45c7a6ef17f8b5,BSI-DSZ-CC-1161-2020
+15fae84c1e6c6c57,ANSSI-CC-2001/05
+eaf0033626287cc4,ANSSI-CC-2012/81
+3357e42cd4dc8605,383-4-256
+4e0dc3133a7e601d,C0513
+8877b69b21e92b41,2019-50-INF-3506
+3c05395ee9e5adf1,BSI-DSZ-CC-0707-2012
+a6e37f57c6e72d8c,ANSSI-CC-2005/13
+f0068ddc5569b772,518-LSS
+04715a4f818f4b71,383-4-345
+3c0185a5cb2d1ea8,CSEC2011001
+f91b60c2954d4b00,CSEC2017005
+b95f2b009d70fad1,BSI-DSZ-CC-0637-2010
+d8c205b4924f91b3,ANSSI-CC-2022/09
+82361ca3849ae025,383-4-420
+40817e88a5a46fd5,CCEVS-VR-VID10541-2014
+c1d88ce9dadd7d2d,BSI-DSZ-CC-0313-2006
+ba27867a74e9c47a,CCEVS-VR-VID10472-2014
+f7cbb33cc639b5b5,KECS-ISIS-0222-2010
+1e7b90184a127017,CCEVS-VR-VID10502-2012,frontpage,
+0d77936727b14c88,BSI-DSZ-CC-0591-2011
+9060684220d85047,ANSSI-CC-2008/01
+b23a595bb30865b5,C0575
+663d89dea15969c8,CRP214
+724d3996cb63d204,CSEC2014001
+5e89afca1d4570a4,C0077
+37c8d23c44b95833,ANSSI-CC-2018/43
+dfb6af00c78cd061,C0421
+b9365d96c9ecdde0,CCEVS-VR-VID10869-2018
+c8427da3733c506d,OCSI/CERT/TEC/05/2017/RC
+780f19df05e0dd18,C0334
+56660124ad6ded6d,BSI-DSZ-CC-1089-2020
+3ba9f330ce93636b,BSI-DSZ-CC-0732-2011
+80932fd09bfc3f98,CCEVS-VR-VID10120-2008
+8298c7814b3b2860,KECS-CISS-1153-2022
+c290ee3692a00006,383-4-438
+642d82f9a5e97fa2,CSEC2017014
+4b2b04554256b523,KECS-ISIS-0117-2008
+4eaceb72085e12f7,C0668
+70b12d34fa092eeb,CCEVS-VR-VID10380-2012
+ec7db499df27dcf9,383-4-397
+61821500cddab34f,BSI-DSZ-CC-0540-2009
+0f3900cdcd0c7f3e,NSCIB-CC-127667-CR3
+238f8edc5eda1358,BSI-DSZ-CC-0285-2005
+b5bcac5eb0f61dfe,CCEVS-VR-10822-2018
+28fb117adbb809e3,CSEC2006002
+c10773ac43ca7c2b,CRP180
+6d6ade44dcc497dd,BSI-DSZ-CC-0312-2005
+c23ca69c2f7528d1,ANSSI-CC-2020/24
+abe89a00b3efa393,C0261
+cf17d0efe1278432,ANSSI-CC-2007/24
+7607e34a7608222c,ISCB-3-RPT-C098-CR-v1
+5ae867bded9ef672,C0180
+3f141b547bd4735f,CCEVS-VR-VID10571-2014
+ed0b1d4c39cbd251,C0727
+abc1f0f55adc2967,CCEVS-VR-VID11049
+d68511e69820f461,CRP260
+59b6bc42781e26d6,CSEC2019006
+d9d0986019774924,KECS-NISS-0050-2006
+318dcee9d8f6ac99,CSEC2018007
+a6fac58198296194,BSI-DSZ-CC-0857-2013
+f1a73b2232787b26,BSI-DSZ-CC-0671-2011
+90f1d708d20deb30,SERTIT-043
+faecd563eb8f1d6c,C0314
+f5d49c63046c69d2,CCEVS-VR-VID10272-2008
+f2e00a1fee27bdd7,BSI-DSZ-CC-0957-2015
+33cb152e0a1be23c,CCEVS-VR-10928-2019
+c16a92b40550193d,383-4-170
+1fb1564dfb0f0b04,ANSSI-CC-2021/26
+8b3c4eb3efd1cdaa,574-LSS
+14ddeb255e0a1cd2,2017-60-INF-2765
+01e3716dd2a5b510,BSI-DSZ-CC-0466-2008
+0c7ef6c32cbdee47,BSI-DSZ-CC-1073-2019
+13e3c6a0cd06ecf6,383-4-340
+814a2ac0aeae08b1,CCEVS-VR-03-0044
+11708925f70ce5d3,Certificate Number: 2011/72
+ca5da2fe138af656,BSI-DSZ-CC-0423-2007
+98937f99289f2499,NSCIB-CC-15-66461-CR
+c3b443f243b95913,CCEVS-VR-11219-2021
+183a5cd3e2aaa0ec,ANSSI-CC-2017/81
\ No newline at end of file
diff --git a/data/cert_id_eval/truth.csv b/data/cert_id_eval/truth.csv
index 0177cc67..80086fc7 100644
--- a/data/cert_id_eval/truth.csv
+++ b/data/cert_id_eval/truth.csv
@@ -98,4 +98,43 @@ b131130935031b4d,Certificate Number: 2002/22,frontpage,y
 dbe20d7c305b24eb,NSCIB-CC-13-37760-CR2,frontpage,y
 6c14f733fe7aa28e,NSCIB-CC-66030-CR5,frontpage,y
 74a8ddae1bb0e894,383-4-10,frontpage,y
-b96dff73aed97fdf,2019-48-INF-3811,frontpage,y
\ No newline at end of file
+b96dff73aed97fdf,2019-48-INF-3811,frontpage,y
+20c98a49ea86a519,CRP153,frontpage,y
+10dc636e9da3e1b6,383-4-349,filename,y
+e2ecf0daf73716f2,511-LSS,frontpage,y
+9219f2fb5f305269,KECS-ISIS-0119-2008,frontpage,y
+10339846e519597c,516-LSS,frontpage,y
+f0068ddc5569b772,518-LSS,frontpage,y
+2d05e976e1adf696,383-4-411,filename,y
+28fb117adbb809e3,CSEC2006002,frontpage,y
+456bfb3665815ccb,KECS-ISIS-0120-2008,frontpage,y
+614f4dcdee0128bd,IC3S/KOL01/ECITelecom/EAL2/0420/0019/CR,frontpage,y
+a79f15874e255007,CRP153,frontpage,y
+bfc86f14e2484335,383-4-430,filename,y
+32d4bf275131aa15,KECS-ISIS-0118-2008,frontpage,y
+1491021aa321b828,517-CCS-2020,frontpage,y
+a91b4c64329d05a8,CSEC2019010,frontpage,y
+c5bad5b309469e22,519-EWA,frontpage,y
+876cb2fe02017f3f,C0012,frontpage,y
+f9c8da9deff77ab5,383-4-384,filename,y
+6266d99d3b983091,383-4-347,filename,y
+4de4a9f436958574,525-LSS-2020,filename,y
+7e664d089ed905a5,383-4-422,frontpage,y
+ae78e8b68bd4c917,CCEVS-VR-CC-11074-2020,frontpage,y
+b61b6299f457a01e,CSEC2006001,frontpage,y
+a5c2f0adcecf1a33,ISCB-5-RPT-C127-CR-v1.1,frontpage,y
+3477723044183b31,ANSSI-CC-2017/42v2-S01,frontpage,y
+d3b7b0df0108a466,383-4-344,filename,y
+6fbb34a3a08e943c,383-4-334,filename,y
+b2aefe8b1bd4d9b3,CRP208A,filename,y
+c312bbf9e303b844,CSEC2019012,frontpage,y
+fb6d7faf0ed2dca5,383-4-362,filename,y
+b3ae94eeccc566ba,383-4-17,frontpage,y
+5f18321fb9a41409,KECS-ISIS-0253-2010,frontpage,y
+a45f04b9a5244a51,KECS-ISIS-0189-2009,frontpage,y
+88eb228eeb487597,KECS-ISIS-0223-2010,frontpage,y
+d5ed93f6586469f6,503-LSS,frontpage,y
+243a29b98f34ae9a,CSEC2019009,frontpage,y
+739f48568b127f22,383-4-346,filename,y
+71e471543ea7e03f,383-4-56,frontpage,y
+f0af2383ad08e3ba,CSEC2019011,frontpage,y
\ No newline at end of file
diff --git a/notebooks/cert_id_eval.ipynb b/notebooks/cert_id_eval.ipynb
index 0d9d004f..c9ed32a8 100644
--- a/notebooks/cert_id_eval.ipynb
+++ b/notebooks/cert_id_eval.ipynb
@@ -33,7 +33,8 @@
    "outputs": [],
    "source": [
     "from sec_certs.dataset import CCDataset\n",
-    "import csv"
+    "import csv\n",
+    "import pandas as pd"
    ]
   },
   {
@@ -42,17 +43,18 @@
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
-    }
+    },
+    "scrolled": true
    },
    "outputs": [],
    "source": [
     "# dset = CCDataset.from_web_latest()\n",
     "dset = CCDataset.from_json(\"../cc_dset/CommonCriteria_dataset.json\")\n",
-    "#dset._extract_report_metadata()\n",
-    "#dset._extract_pdf_frontpage()\n",
+    "#dset._download_reports(False)\n",
+    "#dset._convert_reports_to_txt(False)\n",
     "#dset._extract_report_keywords()\n",
-    "dset._compute_normalized_cert_ids()\n",
-    "dset.to_json()"
+    "#dset._compute_normalized_cert_ids()\n",
+    "#dset.to_json()"
    ]
   },
   {
@@ -80,11 +82,12 @@
    "outputs": [],
    "source": [
     "missing_id_dgsts = set()\n",
+    "missing_id = []\n",
     "for cert in dset:\n",
     "    if not cert.heuristics.cert_id:\n",
-    "        print(cert.dgst, cert.heuristics.cert_id, cert.scheme)\n",
     "        missing_id_dgsts.add(cert.dgst)\n",
-    "print(f\"Total: {len(missing_id_dgsts)}\")"
+    "        missing_id.append((cert.dgst, cert.scheme))\n",
+    "pd.DataFrame(missing_id, columns=[\"id\", \"scheme\"])"
    ]
   },
   {
@@ -98,36 +101,12 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
+    "scrolled": false
    },
    "outputs": [],
    "source": [
-    "i = 0\n",
-    "with open(\"../data/cert_id_eval/missing_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        try:\n",
-    "            cert = dset[line[\"id\"]]\n",
-    "        except:\n",
-    "            continue\n",
-    "        if line[\"cert_id\"] and line[\"cert_id\"] != cert.heuristics.cert_id:\n",
-    "            i += 1\n",
-    "            print(line[\"id\"], line[\"cert_id\"], cert.heuristics.cert_id, line[\"reason\"])\n",
-    "print(f\"Total: {i}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "\n",
-    "The following cell checks which manually analyzed missing certificate IDs were since fixed."
+    "missing_manual = pd.read_csv(\"../data/cert_id_eval/missing_ids.csv\")\n",
+    "set(missing_manual.id) == missing_id_dgsts"
    ]
   },
   {
@@ -140,40 +119,30 @@
    },
    "outputs": [],
    "source": [
-    "manual_missing_ids = set()\n",
-    "i = 0\n",
-    "with open(\"../data/cert_id_eval/missing_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        manual_missing_ids.add(line[\"id\"])\n",
-    "        if line[\"id\"] not in missing_id_dgsts:\n",
-    "            i += 1\n",
-    "            print(\",\".join(line.values()))\n",
-    "print(f\"Total: {i}\")"
+    "num_missing_manual = missing_manual.shape[0]\n",
+    "num_missing_manual_fixable = missing_manual.cert_id.count()\n",
+    "num_missing_manual_unfixable = num_missing_manual - num_missing_manual_fixable\n",
+    "print(f\"\\\\newcommand{{\\\\numccmissingid}}{{{num_missing_manual}}}\")\n",
+    "print(f\"\\\\newcommand{{\\\\numccmissingidfixable}}{{{num_missing_manual_fixable}}}\")\n",
+    "print(f\"\\\\newcommand{{\\\\numccmissingidunfixable}}{{{num_missing_manual_unfixable}}}\")"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "The following cell lists missing certificate IDs that *went missing* since manual analysis."
+    "missing_manual.loc[missing_manual.cert_id.isnull()].reason.value_counts()"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "new_missing_ids = missing_id_dgsts.difference(manual_missing_ids)\n",
-    "for idd in new_missing_ids:\n",
-    "    cert = dset[idd]\n",
-    "    print(idd, cert.heuristics.cert_id, cert.pdf_data.report_filename)\n",
-    "print(f\"Total: {len(new_missing_ids)}\")"
+    "missing_manual.loc[missing_manual.cert_id.notnull()].reason.value_counts()"
    ]
   },
   {
@@ -206,8 +175,7 @@
     "for idd, entries in id_mapping.items():\n",
     "    if len(entries) > 1 and idd:\n",
     "        print(idd, entries)\n",
-    "        duplicate_id_dgsts.update(entries)\n",
-    "print(f\"Total: {len(duplicate_id_dgsts)}\")"
+    "        duplicate_id_dgsts.update(entries)"
    ]
   },
   {
@@ -247,7 +215,9 @@
     "        print(hash, entries)\n",
     "        for entry in entries:\n",
     "            duplicate_doc_dgsts.add(entry)\n",
-    "print(f\"Total: {len(duplicate_doc_dgsts)}\")"
+    "            \n",
+    "duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n",
+    "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)"
    ]
   },
   {
@@ -258,8 +228,7 @@
     }
    },
    "source": [
-    "The following prints the amount of certificate id duplicates due to input data (two or more\n",
-    "certificates share a certification report document)."
+    "The following prints the amount of certificate id duplicates that are not due to input data (and are really our problem)."
    ]
   },
   {
@@ -272,35 +241,19 @@
    },
    "outputs": [],
    "source": [
-    "duplicate_ids_due_doc = duplicate_doc_dgsts.intersection(duplicate_id_dgsts)\n",
-    "print(len(duplicate_ids_due_doc))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
-   "source": [
-    "The following prints the amount of certificate id duplicates that are not due to input data."
+    "for id in duplicate_ids_issue:\n",
+    "    print(id, dset[id].heuristics.cert_id)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "duplicate_ids_issue = duplicate_id_dgsts.difference(duplicate_doc_dgsts)\n",
-    "print(len(duplicate_ids_issue))\n",
-    "for id in duplicate_ids_issue:\n",
-    "    print(id, dset[id].heuristics.cert_id)"
+    "print(f\"\\\\newcommand{{\\\\numccduplicateid}}{{{len(duplicate_id_dgsts)}}}\")\n",
+    "print(f\"\\\\newcommand{{\\\\numccduplicateidcolission}}{{{len(duplicate_ids_due_doc)}}}\")\n",
+    "print(f\"\\\\newcommand{{\\\\numccduplicateidissue}}{{{len(duplicate_ids_issue)}}}\")"
    ]
   },
   {
@@ -311,10 +264,7 @@
     }
    },
    "source": [
-    "### Check manually evaluated duplicates\n",
-    "\n",
-    "The following cell checks that id collisions that were manually analyzed in the past have been fixed.\n",
-    "A `True` means that we have now assigned the correct ID."
+    "### Check manually evaluated duplicates"
    ]
   },
   {
@@ -323,55 +273,31 @@
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
-    }
+    },
+    "scrolled": false
    },
    "outputs": [],
    "source": [
-    "i = 0\n",
-    "with open(\"../data/cert_id_eval/duplicate_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        try:\n",
-    "            cert = dset[line[\"id\"]]\n",
-    "        except:\n",
-    "            continue\n",
-    "        if line[\"true_id\"] != cert.heuristics.cert_id:\n",
-    "            print(line[\"id\"], line[\"result\"], line[\"true_id\"] == cert.heuristics.cert_id , line[\"true_id\"], cert.heuristics.cert_id, line[\"fixable\"])\n",
-    "            i += 1\n",
-    "print(f\"Total: {i}\")"
+    "duplicate_manual = pd.read_csv(\"../data/cert_id_eval/duplicate_ids_new.csv\")\n",
+    "set(duplicate_manual.id) == duplicate_id_dgsts"
    ]
   },
   {
-   "cell_type": "markdown",
-   "metadata": {
-    "pycharm": {
-     "name": "#%% md\n"
-    }
-   },
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "The following cell lists those duplicates that were fixed by changes since manual analysis."
+    "duplicate_manual[duplicate_manual.result == \"tp\"].reason.value_counts()"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "manual_duplicate_ids = set()\n",
-    "i = 0\n",
-    "with open(\"../data/cert_id_eval/duplicate_ids.csv\", \"r\") as f:\n",
-    "    reader = csv.DictReader(f)\n",
-    "    for line in reader:\n",
-    "        manual_duplicate_ids.add(line[\"id\"])\n",
-    "        if line[\"id\"] not in duplicate_id_dgsts:\n",
-    "            i += 1\n",
-    "            print(\",\".join(line.values()))\n",
-    "print(f\"Total: {i}\")"
+    "duplicate_manual[duplicate_manual.result == \"fp\"].reason.value_counts()"
    ]
   },
   {
@@ -382,24 +308,7 @@
     }
    },
    "source": [
-    "The following cell lists duplicates that were *created* since manual analysis."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "new_duplicate_ids = duplicate_id_dgsts.difference(manual_duplicate_ids)\n",
-    "for idd in new_duplicate_ids:\n",
-    "    cert = dset[idd]\n",
-    "    print(idd, cert.heuristics.cert_id)\n",
-    "print(f\"Total: {len(new_duplicate_ids)}\")"
+    "The following cell lists those duplicates that were fixed by changes since manual analysis."
    ]
   },
   {
@@ -416,6 +325,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": [
     "correct = set()\n",
@@ -423,11 +337,9 @@
     "impossible = set()\n",
     "with open(\"../data/cert_id_eval/truth.csv\", \"r\") as f:\n",
     "    reader = csv.DictReader(f)\n",
-    "    i = 0\n",
     "    for line in reader:\n",
-    "        i += 1\n",
-    "        cert = dset[line[\"id\"]]\n",
-    "        if line[\"cert_id\"] != cert.heuristics.cert_id and (line[\"cert_id\"] or cert.heuristics.cert_id is not None):\n",
+    "        cert = dset[line[\"id\"]] # and (line[\"cert_id\"] or cert.heuristics.cert_id is not None)\n",
+    "        if line[\"cert_id\"] != cert.heuristics.cert_id:\n",
     "            print(line[\"id\"], line[\"cert_id\"], cert.heuristics.cert_id, line[\"source\"], line[\"possible\"])\n",
     "            if line[\"possible\"] == \"y\":\n",
     "                possible.add(line[\"id\"])\n",
@@ -436,41 +348,53 @@
     "        else:\n",
     "            correct.add(line[\"id\"])\n",
     "print(len(correct), len(possible), len(impossible))"
-   ],
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "correct = set()\n",
+    "incorrect = set()\n",
+    "with open(\"../data/cert_id_eval/random.csv\", \"r\") as f:\n",
+    "    reader = csv.DictReader(f)\n",
+    "    for line in reader:\n",
+    "        cert = dset[line[\"id\"]]\n",
+    "        if line[\"cert_id\"] != cert.heuristics.cert_id:\n",
+    "            print(line[\"id\"], line[\"cert_id\"], cert.heuristics.cert_id)\n",
+    "            incorrect.add(line[\"id\"])\n",
+    "        else:\n",
+    "            correct.add(line[\"id\"])\n",
+    "print(len(correct), len(incorrect))"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [
-    "for cert in dset:\n",
-    "    if cert.scheme == \"FR\" and cert.dgst in duplicate_ids_issue:\n",
-    "        print(cert.dgst, cert.not_valid_before, cert.heuristics.cert_id, dict(cert.pdf_data.candidate_cert_ids(\"FR\")))"
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "print(f\"\\\\newcommand{{\\\\numccideval}}{{{len(correct) + len(incorrect)}}}\")\n",
+    "print(f\"\\\\newcommand{{\\\\numccidevalcorrect}}{{{len(correct)}}}\")\n",
+    "print(f\"\\\\newcommand{{\\\\numccidevalincorrect}}{{{len(incorrect)}}}\")"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
+   "metadata": {},
    "outputs": [],
-   "source": [],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+   "source": []
   }
  ],
  "metadata": {
@@ -494,4 +418,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 1
-}
\ No newline at end of file
+}
diff --git a/sec_certs/sample/cc_certificate_id.py b/sec_certs/sample/cc_certificate_id.py
index 28247826..28016ee5 100644
--- a/sec_certs/sample/cc_certificate_id.py
+++ b/sec_certs/sample/cc_certificate_id.py
@@ -97,6 +97,8 @@ class CertificateId:
         new_cert_id = self.clean
         if new_cert_id.endswith("-CR"):
             new_cert_id = new_cert_id[:-3]
+        if new_cert_id.endswith("P"):
+            new_cert_id = new_cert_id[:-1]
         return new_cert_id.replace(" ", "-")
 
     def _canonical_jp(self):