diff options
4 files changed, 265 insertions, 1 deletions
diff --git a/src/cz/crcs/ectester/standalone/ECTesterStandalone.java b/src/cz/crcs/ectester/standalone/ECTesterStandalone.java index 034dbc5..0442593 100644 --- a/src/cz/crcs/ectester/standalone/ECTesterStandalone.java +++ b/src/cz/crcs/ectester/standalone/ECTesterStandalone.java @@ -321,7 +321,8 @@ public class ECTesterStandalone { new StandaloneSignatureSuite(null, null, null), new StandaloneCompositeSuite(null, null, null), new StandaloneTwistSuite(null, null, null), - new StandaloneMiscSuite(null, null, null)}; + new StandaloneMiscSuite(null, null, null), + new StandalonePerformanceSuite(null, null, null)}; for (StandaloneTestSuite suite : suites) { System.out.println(" - " + suite.getName()); for (String line : suite.getDescription()) { @@ -782,6 +783,9 @@ public class ECTesterStandalone { case "miscellaneous": suite = new StandaloneMiscSuite(writer, cfg, cli); break; + case "performance": + suite = new StandalonePerformanceSuite(writer, cfg, cli); + break; case "default": default: suite = new StandaloneDefaultSuite(writer, cfg, cli); diff --git a/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java b/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java index 1382c28..7fd1c5a 100644 --- a/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java +++ b/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java @@ -4,6 +4,7 @@ import javax.crypto.KeyAgreement; import javax.crypto.SecretKey; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; import java.security.interfaces.ECPrivateKey; import java.security.interfaces.ECPublicKey; import java.security.spec.AlgorithmParameterSpec; @@ -160,6 +161,14 @@ public class KeyAgreementTestable extends StandaloneTestable<KeyAgreementTestabl hasRun = true; } + @Override + public void reset() { + super.reset(); + try { + ka = KeyAgreement.getInstance(ka.getAlgorithm(), ka.getProvider()); + } catch (NoSuchAlgorithmException e) { } + } + public enum KeyAgreementStage { GetPrivate, GetPublic, diff --git a/src/cz/crcs/ectester/standalone/test/base/PerformanceTest.java b/src/cz/crcs/ectester/standalone/test/base/PerformanceTest.java new file mode 100644 index 0000000..258ca12 --- /dev/null +++ b/src/cz/crcs/ectester/standalone/test/base/PerformanceTest.java @@ -0,0 +1,109 @@ +package cz.crcs.ectester.standalone.test.base; + +import cz.crcs.ectester.common.test.BaseTestable; +import cz.crcs.ectester.common.test.Result; +import cz.crcs.ectester.common.test.SimpleTest; +import cz.crcs.ectester.common.test.TestCallback; + +import java.util.Arrays; + +/** + * @author David Hofman + */ +public class PerformanceTest extends SimpleTest<BaseTestable> { + private long[] times; + private long mean; + private long median; + private long mode; + private final int count; + private final String desc; + + private PerformanceTest(BaseTestable testable, int count, String desc) { + super(testable, new TestCallback<BaseTestable>() { + @Override + public Result apply(BaseTestable testable) { + return new Result(Result.Value.SUCCESS); + } + }); + this.count = count; + this.desc = desc; + } + + public static PerformanceTest repeat(BaseTestable testable, int count) { + return new PerformanceTest(testable, count, null); + } + + public static PerformanceTest repeat(BaseTestable testable, String desc, int count) { + return new PerformanceTest(testable, count, desc); + } + + @Override + public String getDescription() { + String rest = String.format("Mean = %d ns, Median = %d ns, Mode = %d ns", mean, median, mode); + return (desc == null ? rest : desc + " (" + rest + ")"); + } + + @Override + protected void runSelf() { + + times = new long[count]; + for (int i = 0; i < count; ++i) { + times[i] = measureTime(); + } + + mean = Arrays.stream(times).sum() / count; + + long[] sorted = times.clone(); + Arrays.sort(sorted); + if (count % 2 == 0) { + median = (sorted[(count / 2) - 1] + sorted[count / 2]) / 2; + } else { + median = sorted[count / 2]; + } + + long max_occurrences = 0; + int i = 0; + while (i < count) { + long current_value = sorted[i]; + long current_occurrences = 0; + while (i < count && sorted[i] == current_value) { + i++; + current_occurrences++; + } + if (current_occurrences > max_occurrences) { + max_occurrences = current_occurrences; + mode = current_value; + } + } + result = callback.apply(testable); + } + + public long getCount() { + return count; + } + + public long[] getTimes() { + return times; + } + + public long getMean() { + return mean; + } + + public long getMedian() { + return median; + } + + public long getMode() { + return mode; + } + + private long measureTime() { + if(testable.hasRun()) { + testable.reset(); + } + long startTime = System.nanoTime(); + testable.run(); + return System.nanoTime() - startTime; + } +} diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandalonePerformanceSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandalonePerformanceSuite.java new file mode 100644 index 0000000..dd50862 --- /dev/null +++ b/src/cz/crcs/ectester/standalone/test/suites/StandalonePerformanceSuite.java @@ -0,0 +1,142 @@ +package cz.crcs.ectester.standalone.test.suites; + +import cz.crcs.ectester.common.cli.TreeCommandLine; +import cz.crcs.ectester.common.ec.EC_Curve; +import cz.crcs.ectester.common.output.TestWriter; +import cz.crcs.ectester.common.test.CompoundTest; +import cz.crcs.ectester.common.test.Result; +import cz.crcs.ectester.common.test.Test; +import cz.crcs.ectester.data.EC_Store; +import cz.crcs.ectester.standalone.ECTesterStandalone; +import cz.crcs.ectester.standalone.consts.KeyAgreementIdent; +import cz.crcs.ectester.standalone.consts.KeyPairGeneratorIdent; +import cz.crcs.ectester.standalone.consts.SignatureIdent; +import cz.crcs.ectester.standalone.test.base.*; + +import javax.crypto.KeyAgreement; +import java.security.KeyPairGenerator; +import java.security.Signature; +import java.security.interfaces.ECPrivateKey; +import java.security.spec.ECParameterSpec; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author David Hofman + */ +public class StandalonePerformanceSuite extends StandaloneTestSuite { + private final int count = 100; + + public StandalonePerformanceSuite(TestWriter writer, ECTesterStandalone.Config cfg, TreeCommandLine cli) { + super(writer, cfg, cli, "performance", "The performance test suite measures performance of KeyPair generation, KeyAgreement and Signature operations.", + "Supports options:", + "\t - gt/kpg-type (select multiple types by separating them with commas)", + "\t - kt/ka-type (select multiple types by separating them with commas)", + "\t - st/sig-type (select multiple types by separating them with commas)", + "\t - key-type"); + } + + @Override + protected void runTests() throws Exception { + String kpgAlgo = cli.getOptionValue("test.kpg-type"); + String kaAlgo = cli.getOptionValue("test.ka-type"); + String sigAlgo = cli.getOptionValue("test.sig-type"); + String keyAlgo = cli.getOptionValue("test.key-type", "AES"); + + List<String> kpgTypes = kpgAlgo != null ? Arrays.asList(kpgAlgo.split(",")) : new ArrayList<>(); + List<String> kaTypes = kaAlgo != null ? Arrays.asList(kaAlgo.split(",")) : new ArrayList<>(); + List<String> sigTypes = sigAlgo != null ? Arrays.asList(sigAlgo.split(",")) : new ArrayList<>(); + + List<KeyPairGeneratorIdent> kpgIdents = new LinkedList<>(); + if (kpgAlgo == null) { + // try EC, if not, fail with: need to specify kpg algo. + Optional<KeyPairGeneratorIdent> kpgIdentOpt = cfg.selected.getKPGs().stream() + .filter((ident) -> ident.contains("EC")) + .findFirst(); + if (kpgIdentOpt.isPresent()) { + kpgIdents.add(kpgIdentOpt.get()); + } else { + System.err.println("The default KeyPairGenerator algorithm type of \"EC\" was not found. Need to specify a type."); + return; + } + } else { + // try the specified, if not, fail with: wrong kpg algo/not found. + kpgIdents = cfg.selected.getKPGs().stream() + .filter((ident) -> ident.containsAny(kpgTypes)).collect(Collectors.toList()); + if (kpgIdents.isEmpty()) { + System.err.println("No KeyPairGenerator algorithms of specified types were found."); + return; + } + } + + KeyGeneratorTestable kgtOne = null; + KeyGeneratorTestable kgtOther = null; + ECParameterSpec spec = null; + List<Test> kpgTests = new LinkedList<>(); + for(KeyPairGeneratorIdent kpgIdent : kpgIdents) { + KeyPairGenerator kpg = kpgIdent.getInstance(cfg.selected.getProvider()); + if (cli.hasOption("test.bits")) { + int bits = Integer.parseInt(cli.getOptionValue("test.bits")); + kgtOne = new KeyGeneratorTestable(kpg, bits); + kgtOther = new KeyGeneratorTestable(kpg, bits); + } else if (cli.hasOption("test.named-curve")) { + String curveName = cli.getOptionValue("test.named-curve"); + EC_Curve curve = EC_Store.getInstance().getObject(EC_Curve.class, curveName); + if (curve == null) { + System.err.println("Curve not found: " + curveName); + return; + } + spec = curve.toSpec(); + kgtOne = new KeyGeneratorTestable(kpg, spec); + kgtOther = new KeyGeneratorTestable(kpg, spec); + } else { + kgtOne = new KeyGeneratorTestable(kpg); + kgtOther = new KeyGeneratorTestable(kpg); + } + kpgTests.add(PerformanceTest.repeat(kgtOne, kpgIdent.getName(), count)); + } + runTest(KeyGeneratorTest.expect(kgtOther, Result.ExpectedValue.SUCCESS)); + doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "KeyPairGenerator performance tests", kpgTests.toArray(new Test[0]))); + + List<Test> kaTests = new LinkedList<>(); + for (KeyAgreementIdent kaIdent : cfg.selected.getKAs()) { + if (kaAlgo == null || kaIdent.containsAny(kaTypes)) { + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); + KeyAgreementTestable testable; + if (kaIdent.requiresKeyAlgo()) { + testable = new KeyAgreementTestable(ka, kgtOne, kgtOther, spec, keyAlgo); + } else { + testable = new KeyAgreementTestable(ka, kgtOne, kgtOther, spec); + } + kaTests.add(PerformanceTest.repeat(testable, kaIdent.getName(), count)); + } + } + if(kaTests.isEmpty()) { + kaTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, "None of the specified KeyAgreement types is supported by the library.")); + } + doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "KeyAgreement performance tests", kaTests.toArray(new Test[0]))); + + List<Test> sigTests = new LinkedList<>(); + List<Test> sigTestsNoVerification = new LinkedList<>(); + for (SignatureIdent sigIdent : cfg.selected.getSigs()) { + if (sigAlgo == null || sigIdent.containsAny(sigTypes)) { + Signature sig = sigIdent.getInstance(cfg.selected.getProvider()); + sigTests.add(PerformanceTest.repeat(new SignatureTestable(sig, kgtOne, null), sigIdent.getName(),count)); + if(kgtOne.getKeyPair() != null) { + ECPrivateKey signKey = (ECPrivateKey) kgtOne.getKeyPair().getPrivate(); + sigTestsNoVerification.add(PerformanceTest.repeat(new SignatureTestable(sig, signKey, null, null), sigIdent.getName(), count)); + } + } + } + if(sigTestsNoVerification.isEmpty() & !sigTests.isEmpty()) { + sigTestsNoVerification.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Signature tests with no verification require a successfully generated private key.")); + } + if(sigTests.isEmpty()) { + sigTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, "None of the specified Signature types is supported by the library.")); + sigTestsNoVerification.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, "None of the specified Signature types is supported by the library.")); + } + Test signAndVerify = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Sign and verify", sigTests.toArray(new Test[0])); + Test signOnly = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Sign only, no verification", sigTestsNoVerification.toArray(new Test[0])); + doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Signature performance tests", signAndVerify, signOnly)); + } +} |