1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
#!/usr/bin/env python
import argparse
import json
import time
import os
from pathlib import Path
import subprocess as sp
def library_name(library):
match library:
case "cryptopp":
return "Crypto++"
case "boringssl":
return "BoringSSL"
case "openssl":
return "OpenSSL"
case "botan":
return "Botan"
case "ippcp":
return "IPPCP"
case "libressl":
return "LibreSSL"
case "nettle":
return "Nettle"
case "gcrypt":
return "libgcrypt"
case "mbedtls":
return "mbedTLS"
def base_options(library):
match library:
case "openssl" | "botan" | "boringssl" | "ippcp" | "libressl" | "gcrypt" | "nettle":
return ["-ps", "123412341234123412341234123412341234123412341234123412341234123412341234123412341234123412341234"]
case "cryptopp" | "mbedtls":
return ["-ps", "12345678"]
case _:
return []
def default_options(library):
match library:
case "botan" | "cryptopp":
return ["-gt", "ECDH"]
case _:
return []
def test_vectors_options(library):
return default_options(library)
def performance_options(library):
return default_options(library)
def signature_options(library):
match library:
case "nettle" | "gcrypt" | "boringssl" | "openssl" | "tomcrypt" | "libressl" | "ippcp" | "mbedtls":
return ["-st", "NONEwithECDSA"]
case _:
return []
def miscellaneous_options(library):
return default_options(library)
def twist_options(library):
return default_options(library)
def invalid_options(library):
return default_options(library)
def degenerate_options(library):
return default_options(library)
def cofactor_options(library):
return default_options(library)
def composite_options(library):
return default_options(library)
def edge_cases_options(library):
return default_options(library)
def wrong_options(library):
return default_options(library)
def build_library(library, version):
command = ["nix", "build", "--log-format", "raw", f"?submodules=1#{library}.{version}"]
result = sp.run(command, check=False)
print(f"build {library} {version} = {result.returncode}")
return result.returncode == 0
def test_library(library, test_suite, version):
opts = base_options(library)
opts.extend(globals()[f"{test_suite.replace('-', '_')}_options"](library))
lib_name = library_name(library)
command = ["./result/bin/ECTesterStandalone", "test",
f"-oyml:results/yml/{library}_{test_suite}_{version}.yml",
f"-otext:results/txt/{library}_{test_suite}_{version}.txt",
f"-oxml:results/xml/{library}_{test_suite}_{version}.xml",
"-q", *opts, test_suite, lib_name]
try:
result = sp.run(command, timeout=60, check=False)
print(f"run {library} {test_suite} {version} = {result.returncode}")
except sp.TimeoutExpired:
print(f"run {library} {test_suite} {version} timed-out!")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-l", "--library")
parser.add_argument("-s", "--suite")
args = parser.parse_args()
library = args.library
suite = args.suite
libraries = [
"botan",
"cryptopp",
"openssl",
"boringssl",
"gcrypt",
"mbedtls",
"ippcp",
"nettle",
"libressl",
]
suites = [
"default",
"test-vectors",
"performance",
"signature",
"miscellaneous",
"invalid",
"twist",
"degenerate",
"edge-cases",
"cofactor",
"composite",
"wrong"
]
if library is None:
libraries2test = libraries
else:
libraries2test = [library]
if suite is None:
suites2test = suites
else:
suites2test = [suite]
os.makedirs("results/yml/", exist_ok=True)
os.makedirs("results/txt/", exist_ok=True)
os.makedirs("results/xml/", exist_ok=True)
for library in libraries2test:
with open(f"./nix/{library}_pkg_versions.json", "r") as f:
versions = list(json.load(f).keys())
for version in versions:
built = build_library(library, version)
if built:
for suite in suites2test:
test_library(library, suite, version)
if __name__ == '__main__':
main()
|
