diff options
9 files changed, 38 insertions, 13 deletions
@@ -105,6 +105,10 @@ To select which tests will be performed, it is possible to enter the test suite which specifies the number of the first test to be run, and optionally the number of the last test to be run as `-t <test_suite>[:start_index[:stop_index]]`. Use with `-o / --output [out_type:]<out_file>` to output the test results to a file. +Use `-n / --number` to repeat tests multiple times (for example if you want more information on timings). +Use `-ts / --test-shuffle` to shuffle the tests in the test suite. +Use `--cleanup` to issue a JavaCard cleanup command after each group of tests (`JCSystem.requestObjectDeletion()`). + For possible formats of this file see [FORMAT](docs/FORMAT.md). For more info about the test suites see [TESTS](docs/TESTS.md). @@ -122,6 +126,7 @@ Generates batch of EC keypairs and exports them. Use with `-o / --output [out_file]` to output the generated keys to a file. Use with `--time` to measure time as a difference of real duration of the operation and the dry-run duration of the operation. +Use `--time-unit [milli, micro, nano]` to specify the time unit of the output. For format of this file see [FORMAT](docs/FORMAT.md). #### ECDH @@ -131,6 +136,7 @@ Performs ECDH. Use with `-o / --output [out_file]` to output into a file. Use with `--time` to measure time as a difference of real duration of the operation and the dry-run duration of the operation. +Use `--time-unit [milli, micro, nano]` to specify the time unit of the output. For format of this file see [FORMAT](docs/FORMAT.md). Respects the KeyAgreement type specified in `-ka / --ka-type [type]`. @@ -142,6 +148,7 @@ Performs ECDSA. Useful with `-i / --input [in_file]` to sign the contents of a file. Use with `-o / --output [out_file]` to output into a file. Use with `--time` to measure time as a difference of real duration of the operation and the dry-run duration of the operation. +Use `--time-unit [milli, micro, nano]` to specify the time unit of the output. For format of these files see [FORMAT](docs/FORMAT.md). Respects the Signature type specified in `-sig / --sig-type [type]`. diff --git a/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java b/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java index 082be23..ddd733b 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java @@ -341,9 +341,10 @@ public class ECTesterReader { opts.addOption(Option.builder().longOpt("time").desc("Output better timing values, by running command in dry run mode and normal mode, and subtracting the two.").build()); opts.addOption(Option.builder().longOpt("time-unit").desc("Use given time unit in measurement, one of: milli, micro, nano.").hasArg().argName("unit").build()); opts.addOption(Option.builder().longOpt("cleanup").desc("Send the cleanup command triggering JCSystem.requestObjectDeletion() after some operations.").build()); - opts.addOption(Option.builder("n").longOpt("number").desc("Number of repeats during testing.").hasArg().argName("number").build()); opts.addOption(Option.builder("s").longOpt("simulate").desc("Simulate a card with jcardsim instead of using a terminal.").build()); opts.addOption(Option.builder("y").longOpt("yes").desc("Accept all warnings and prompts.").build()); + opts.addOption(Option.builder("n").longOpt("number").desc("Number of repeats during testing.").hasArg().argName("number").build()); + opts.addOption(Option.builder("ts").longOpt("test-shuffle").desc("Shuffle the test suite before running it.").build()); opts.addOption(Option.builder("tk").longOpt("test-key").desc("Key setup technique to use in test suites:\n- generate (default): Generate keypairs on the card.\n- deterministic: Prepare keypairs deterministically off-card.\n- random: Prepare keypairs randomly off-card.").hasArg().argName("option").build()); opts.addOption(Option.builder("td").longOpt("test-data").desc("Data setup technique to use in test suites:\n- random (default): Prepare data randomly off-card.\n- deterministic: Prepare data deterministically off-card.").hasArg().argName("option").build()); @@ -853,7 +854,6 @@ public class ECTesterReader { public String timeUnit; public boolean cleanup = false; public boolean simulate = false; - public int number = 1; public boolean yes = false; public String format; public boolean color; @@ -868,6 +868,8 @@ public class ECTesterReader { public byte ECKAType = EC_Consts.KeyAgreement_ALG_EC_SVDP_DH; public int ECDSACount; public byte ECDSAType = EC_Consts.Signature_ALG_ECDSA_SHA; + public int number = 1; + public boolean testShuffle; public String testKeySetup; public String testDataSetup; @@ -916,7 +918,6 @@ public class ECTesterReader { time = cli.hasOption("time"); cleanup = cli.hasOption("cleanup"); simulate = cli.hasOption("simulate"); - number = Integer.parseInt(cli.getOptionValue("number", "1")); yes = cli.hasOption("yes"); color = cli.hasOption("color"); Colors.enabled = color; @@ -1000,6 +1001,9 @@ public class ECTesterReader { primeField = true; } + number = Integer.parseInt(cli.getOptionValue("number", "1")); + testShuffle = cli.hasOption("test-shuffle"); + String suiteOpt = cli.getOptionValue("test", "default").toLowerCase(); if (suiteOpt.contains(":")) { String[] parts = suiteOpt.split(":"); diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java index 1ed91db..01d278e 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java @@ -54,7 +54,8 @@ public class CardCofactorSuite extends CardTestSuite { ecdhTests.add(CompoundTest.all(ExpectedValue.SUCCESS, pub.getId() + " cofactor key test.", objectEcdh, rawEcdh)); } } - Collections.shuffle(ecdhTests); + if (cfg.testShuffle) + Collections.shuffle(ecdhTests); Test ecdh = CompoundTest.all(ExpectedValue.SUCCESS, "Perform ECDH with public points on non-generator subgroup.", ecdhTests.toArray(new Test[0])); if (cfg.cleanup) { diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java index 71890e1..7e442b0 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java @@ -12,6 +12,7 @@ import cz.crcs.ectester.reader.CardMngr; import cz.crcs.ectester.reader.ECTesterReader; import cz.crcs.ectester.reader.command.Command; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -44,8 +45,12 @@ public class CardCompositeSuite extends CardTestSuite { for (EC_Key key : curveKeys.getValue()) { Command ecdhCommand = new Command.ECDH_direct(this.card, CardConsts.KEYPAIR_LOCAL, CardConsts.EXPORT_FALSE, EC_Consts.TRANSFORMATION_NONE, EC_Consts.KeyAgreement_ALG_EC_SVDP_DH, key.flatten()); Test ecdh = CommandTest.expect(ecdhCommand, ExpectedValue.FAILURE, "Card correctly rejected to do ECDH over a composite order curve.", "Card incorrectly does ECDH over a composite order curve, leaks bits of private key."); - tests.add(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Composite test of " + curve.getId() + ", with " + key.getDesc(), ecdh)); + for (int i = 0; i < cfg.number; ++i) { + tests.add(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Composite test of " + curve.getId() + ", with " + key.getDesc(), ecdh)); + } } + if (cfg.testShuffle) + Collections.shuffle(tests); Test ecdhTest = CompoundTest.all(ExpectedValue.SUCCESS, "Do ECDH.", tests.toArray(new Test[0])); doTest(CompoundTest.greedyAll(ExpectedValue.SUCCESS, "Composite test of " + curve.getId() + ".", prepare, ecdhTest)); } diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java index aa69565..1bef698 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java @@ -55,7 +55,8 @@ public class CardDegenerateSuite extends CardTestSuite { } //TODO: actually get the result of ECDH here, as well as export privkey and compare to exponentiation in Fp^*. } - Collections.shuffle(ecdhTests); + if (cfg.testShuffle) + Collections.shuffle(ecdhTests); Test ecdh = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Perform ECDH with degenerate public points.", ecdhTests.toArray(new Test[0])); if (cfg.cleanup) { Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY); diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java index ed24e59..787c1c7 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java @@ -101,7 +101,8 @@ public class CardEdgeCasesSuite extends CardTestSuite { curveTests.add(one); } } - Collections.shuffle(curveTests); + if (cfg.testShuffle) + Collections.shuffle(curveTests); if (cfg.cleanup) { curveTests.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY)); @@ -236,7 +237,8 @@ public class CardEdgeCasesSuite extends CardTestSuite { for (int i = 0; i < cfg.number; ++i) { tests.addAll(Arrays.asList(zeroS, oneS, alternateS, alternateOtherS, fullS, smallerS, exactS, largerS, rm1S, rp1S, krS, krm1S, krp1S)); } - Collections.shuffle(tests); + if (cfg.testShuffle) + Collections.shuffle(tests); if (cfg.cleanup) { Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY); @@ -312,7 +314,8 @@ public class CardEdgeCasesSuite extends CardTestSuite { for (int j = 0; j < cfg.number; ++j) { tests160.addAll(Arrays.asList(zeroTest, pTest, rTest)); } - Collections.shuffle(tests160); + if (cfg.testShuffle) + Collections.shuffle(tests160); tests160.add(0, setup); if (cfg.cleanup) { tests160.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY)); diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java index 69c1e52..2cb8e11 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java @@ -58,7 +58,8 @@ public class CardInvalidSuite extends CardTestSuite { ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " invalid key test.", objectEcdh, rawEcdh)); } } - Collections.shuffle(ecdhTests); + if (cfg.testShuffle) + Collections.shuffle(ecdhTests); Test ecdh = CompoundTest.all(ExpectedValue.SUCCESS, "Perform ECDH with invalid public points.", ecdhTests.toArray(new Test[0])); if (cfg.cleanup) { diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java index 3cda557..e61be83 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java @@ -36,7 +36,8 @@ public class CardSignatureSuite extends CardTestSuite { byte[] data = "Some stuff that is not the actual data".getBytes(); for (int i = 0; i < cfg.number; ++i) { - Collections.shuffle(nok); + if (cfg.testShuffle) + Collections.shuffle(nok); for (EC_SigResult sig : nok) { ecdsaTest(sig, Result.ExpectedValue.FAILURE, data); } @@ -44,7 +45,8 @@ public class CardSignatureSuite extends CardTestSuite { List<EC_SigResult> ok = groups.entrySet().stream().filter((e) -> e.getKey().equals("ok")).findFirst().get().getValue(); for (int i = 0; i < cfg.number; ++i) { - Collections.shuffle(ok); + if (cfg.testShuffle) + Collections.shuffle(ok); for (EC_SigResult sig : ok) { ecdsaTest(sig, Result.ExpectedValue.SUCCESS, null); } diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java index 19b9ddd..1f7f222 100644 --- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java +++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java @@ -52,7 +52,8 @@ public class CardTwistSuite extends CardTestSuite { ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " twist key test.", objectEcdh, rawEcdh)); } } - Collections.shuffle(ecdhTests); + if (cfg.testShuffle) + Collections.shuffle(ecdhTests); Test ecdh = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Perform ECDH with public points on twist.", ecdhTests.toArray(new Test[0])); Test tests = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Do tests.", ecdh); |