7 files changed, 63 insertions, 17 deletions

diff --git a/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java b/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java
index 6f624a0..082be23 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java
@@ -324,10 +324,10 @@ public class ECTesterReader {
 
         OptionGroup key = new OptionGroup();
         key.addOption(Option.builder("nk").longOpt("named-key").desc("Use keyPair from KeyDB: <cat/id>").hasArg().argName("cat/id").build());
-        key.addOption(Option.builder("k").longOpt("key").desc("Use keyPair from file <key_file> (wx,wy,s).").hasArg().argName("key_file").build());
+        key.addOption(Option.builder("k").longOpt("key").desc("Use keyPair from file <key_file> (wx,wy,s).").hasArg().argName("key_file").build());
         opts.addOptionGroup(key);
 
-        opts.addOption(Option.builder("i").longOpt("input").desc("Input from file <input_file>, for ECDSA signing.").hasArg().argName("input_file").build());
+        opts.addOption(Option.builder("i").longOpt("input").desc("Input from file <input_file>, for ECDSA signing.").hasArg().argName("input_file").build());
         opts.addOption(Option.builder("o").longOpt("output").desc("Output into file <output_file>. The file can be prefixed by the format (one of text,yml,xml), such as: xml:<output_file>.").hasArgs().argName("output_file").build());
         opts.addOption(Option.builder("l").longOpt("log").desc("Log output into file [log_file].").hasArg().argName("log_file").optionalArg(true).build());
         opts.addOption(Option.builder("v").longOpt("verbose").desc("Turn on verbose logging.").build());
@@ -341,6 +341,7 @@ public class ECTesterReader {
         opts.addOption(Option.builder().longOpt("time").desc("Output better timing values, by running command in dry run mode and normal mode, and subtracting the two.").build());
         opts.addOption(Option.builder().longOpt("time-unit").desc("Use given time unit in measurement, one of: milli, micro, nano.").hasArg().argName("unit").build());
         opts.addOption(Option.builder().longOpt("cleanup").desc("Send the cleanup command triggering JCSystem.requestObjectDeletion() after some operations.").build());
+        opts.addOption(Option.builder("n").longOpt("number").desc("Number of repeats during testing.").hasArg().argName("number").build());
         opts.addOption(Option.builder("s").longOpt("simulate").desc("Simulate a card with jcardsim instead of using a terminal.").build());
         opts.addOption(Option.builder("y").longOpt("yes").desc("Accept all warnings and prompts.").build());
         opts.addOption(Option.builder("tk").longOpt("test-key").desc("Key setup technique to use in test suites:\n- generate (default): Generate keypairs on the card.\n- deterministic: Prepare keypairs deterministically off-card.\n- random: Prepare keypairs randomly off-card.").hasArg().argName("option").build());
@@ -852,6 +853,7 @@ public class ECTesterReader {
         public String timeUnit;
         public boolean cleanup = false;
         public boolean simulate = false;
+        public int number = 1;
         public boolean yes = false;
         public String format;
         public boolean color;
@@ -914,6 +916,7 @@ public class ECTesterReader {
             time = cli.hasOption("time");
             cleanup = cli.hasOption("cleanup");
             simulate = cli.hasOption("simulate");
+            number = Integer.parseInt(cli.getOptionValue("number", "1"));
             yes = cli.hasOption("yes");
             color = cli.hasOption("color");
             Colors.enabled = color;
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java
index b0ad2d3..1ed91db 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java
@@ -1,18 +1,19 @@
 package cz.crcs.ectester.reader.test;
 
+import cz.crcs.ectester.common.ec.EC_Consts;
 import cz.crcs.ectester.common.ec.EC_Curve;
 import cz.crcs.ectester.common.ec.EC_Key;
-import cz.crcs.ectester.common.ec.EC_Consts;
 import cz.crcs.ectester.common.output.TestWriter;
 import cz.crcs.ectester.common.test.CompoundTest;
 import cz.crcs.ectester.common.test.Test;
-import cz.crcs.ectester.common.util.CardUtil;
 import cz.crcs.ectester.common.util.CardConsts;
+import cz.crcs.ectester.common.util.CardUtil;
 import cz.crcs.ectester.data.EC_Store;
 import cz.crcs.ectester.reader.CardMngr;
 import cz.crcs.ectester.reader.ECTesterReader;
 import cz.crcs.ectester.reader.command.Command;
 
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -49,8 +50,11 @@ public class CardCofactorSuite extends CardTestSuite {
                 Test objectEcdh = CompoundTest.any(ExpectedValue.SUCCESS, CardUtil.getKATypeString(EC_Consts.KeyAgreement_ALG_EC_SVDP_DH) + " test with cofactor pubkey.", setPub, ecdh);
                 Command ecdhCommand = new Command.ECDH_direct(this.card, CardConsts.KEYPAIR_LOCAL, CardConsts.EXPORT_FALSE, EC_Consts.TRANSFORMATION_NONE, EC_Consts.KeyAgreement_ALG_EC_SVDP_DH, pub.flatten());
                 Test rawEcdh = CommandTest.expect(ecdhCommand, ExpectedValue.FAILURE, "Card correctly rejected point on non-generator subgroup.", "Card incorrectly accepted point on non-generator subgroup.");
-                ecdhTests.add(CompoundTest.all(ExpectedValue.SUCCESS, pub.getId() + " cofactor key test.", objectEcdh, rawEcdh));
+                for (int i = 0; i < cfg.number; ++i) {
+                    ecdhTests.add(CompoundTest.all(ExpectedValue.SUCCESS, pub.getId() + " cofactor key test.", objectEcdh, rawEcdh));
+                }
             }
+            Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(ExpectedValue.SUCCESS, "Perform ECDH with public points on non-generator subgroup.", ecdhTests.toArray(new Test[0]));
 
             if (cfg.cleanup) {
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
index 6817390..aa69565 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
@@ -14,6 +14,7 @@ import cz.crcs.ectester.reader.CardMngr;
 import cz.crcs.ectester.reader.ECTesterReader;
 import cz.crcs.ectester.reader.command.Command;
 
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -49,9 +50,12 @@ public class CardDegenerateSuite extends CardTestSuite {
                 Test objectEcdh = CompoundTest.any(Result.ExpectedValue.SUCCESS, CardUtil.getKATypeString(EC_Consts.KeyAgreement_ALG_EC_SVDP_DH) + " test with degenerate pubkey.", setPub, ecdh);
                 Command ecdhCommand = new Command.ECDH_direct(this.card, CardConsts.KEYPAIR_LOCAL, CardConsts.EXPORT_TRUE, EC_Consts.TRANSFORMATION_NONE, EC_Consts.KeyAgreement_ALG_EC_SVDP_DH, pub.flatten());
                 Test rawEcdh = CommandTest.expect(ecdhCommand, Result.ExpectedValue.FAILURE, "Card correctly rejected point on degenerate curve.", "Card incorrectly accepted point on degenerate curve.");
-                ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " degenerate key test.", objectEcdh, rawEcdh));
+                for (int i = 0; i < cfg.number; ++i) {
+                    ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " degenerate key test.", objectEcdh, rawEcdh));
+                }
                 //TODO: actually get the result of ECDH here, as well as export privkey and compare to exponentiation in Fp^*.
             }
+            Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Perform ECDH with degenerate public points.", ecdhTests.toArray(new Test[0]));
             if (cfg.cleanup) {
                 Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY);
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java
index 00bc34e..cfae0cc 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java
@@ -97,8 +97,11 @@ public class CardEdgeCasesSuite extends CardTestSuite {
                     Test ka = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Do", ecdhPreTest, ecdh);
 
                     Test one = CompoundTest.greedyAllTry(Result.ExpectedValue.SUCCESS, "Test " + id + ".", prepare, ka);
-                    curveTests.add(one);
+                    for (int i = 0; i < cfg.number; ++i) {
+                        curveTests.add(one);
+                    }
                 }
+                Collections.shuffle(curveTests);
 
                 if (cfg.cleanup) {
                     curveTests.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY));
@@ -229,12 +232,18 @@ public class CardEdgeCasesSuite extends CardTestSuite {
             EC_Params krp1Params = makeParams(krp1);
             Test krp1S = ecdhTest(new Command.Set(this.card, CardConsts.KEYPAIR_REMOTE, EC_Consts.CURVE_external, krp1Params.getParams(), krp1Params.flatten()), "ECDH with S = (k * r) + 1.", Result.ExpectedValue.ANY, Result.ExpectedValue.ANY);
 
+            List<Test> tests = new LinkedList<>();
+            for (int i = 0; i < cfg.number; ++i) {
+                tests.addAll(Arrays.asList(zeroS, oneS, alternateS, alternateOtherS, fullS, smallerS, exactS, largerS, rm1S, rp1S, krS, krm1S, krp1S));
+            }
+            Collections.shuffle(tests);
+
             if (cfg.cleanup) {
                 Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY);
-                doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Tests with edge-case private key values over " + curve.getId() + ".", setup, zeroS, oneS, alternateS, alternateOtherS, fullS, smallerS, exactS, largerS, rm1S, rp1S, krS, krm1S, krp1S, cleanup));
-            } else {
-                doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Tests with edge-case private key values over " + curve.getId() + ".", setup, zeroS, oneS, alternateS, alternateOtherS, fullS, smallerS, exactS, largerS, rm1S, rp1S, krS, krm1S, krp1S));
+                tests.add(cleanup);
             }
+            tests.addFirst(setup);
+            doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Tests with edge-case private key values over " + curve.getId() + ".", tests.toArray(new Test[0])));
         }
 
         EC_Curve secp160r1 = EC_Store.getInstance().getObject(EC_Curve.class, "secg/secp160r1");
@@ -298,7 +307,17 @@ public class CardEdgeCasesSuite extends CardTestSuite {
             }
         }
         Test rTest = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Near r.", rTests);
-        doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test private key values near zero, near p and near/larger than the order.", setup, zeroTest, pTest, rTest));
+
+        List<Test> tests160 = new LinkedList<>();
+        for (int j = 0; j < cfg.number; ++j) {
+            tests160.addAll(Arrays.asList(zeroTest, pTest, rTest));
+        }
+        Collections.shuffle(tests160);
+        tests160.addFirst(setup);
+        if (cfg.cleanup) {
+            tests160.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY));
+        }
+        doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test private key values near zero, near p and near/larger than the order.", tests160.toArray(new Test[0])));
     }
 
     private Test ecdhTestBoth(Command setPriv, String desc, Result.ExpectedValue setExpect, Result.ExpectedValue ecdhExpect) {
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java
index cacfba9..69c1e52 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java
@@ -14,6 +14,7 @@ import cz.crcs.ectester.reader.CardMngr;
 import cz.crcs.ectester.reader.ECTesterReader;
 import cz.crcs.ectester.reader.command.Command;
 
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -53,8 +54,11 @@ public class CardInvalidSuite extends CardTestSuite {
                 Test objectEcdh = CompoundTest.any(Result.ExpectedValue.SUCCESS, CardUtil.getKATypeString(EC_Consts.KeyAgreement_ALG_EC_SVDP_DH) + " test with invalid pubkey.", setPub, ecdh);
                 Command ecdhCommand = new Command.ECDH_direct(this.card, CardConsts.KEYPAIR_LOCAL, CardConsts.EXPORT_FALSE, EC_Consts.TRANSFORMATION_NONE, EC_Consts.KeyAgreement_ALG_EC_SVDP_DH, pub.flatten());
                 Test rawEcdh = CommandTest.expect(ecdhCommand, ExpectedValue.FAILURE, "Card correctly rejected point on invalid curve.", "Card incorrectly accepted point on invalid curve.");
-                ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " invalid key test.", objectEcdh, rawEcdh));
+                for (int i = 0; i < cfg.number; ++i) {
+                    ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " invalid key test.", objectEcdh, rawEcdh));
+                }
             }
+            Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(ExpectedValue.SUCCESS, "Perform ECDH with invalid public points.", ecdhTests.toArray(new Test[0]));
 
             if (cfg.cleanup) {
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java
index 0bf897a..3cda557 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java
@@ -14,6 +14,8 @@ import cz.crcs.ectester.reader.CardMngr;
 import cz.crcs.ectester.reader.ECTesterReader;
 import cz.crcs.ectester.reader.command.Command;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -33,13 +35,19 @@ public class CardSignatureSuite extends CardTestSuite {
         List<EC_SigResult> nok = groups.entrySet().stream().filter((e) -> e.getKey().equals("nok")).findFirst().get().getValue();
 
         byte[] data = "Some stuff that is not the actual data".getBytes();
-        for (EC_SigResult sig : nok) {
-            ecdsaTest(sig, Result.ExpectedValue.FAILURE, data);
+        for (int i = 0; i < cfg.number; ++i) {
+            Collections.shuffle(nok);
+            for (EC_SigResult sig : nok) {
+                ecdsaTest(sig, Result.ExpectedValue.FAILURE, data);
+            }
         }
 
         List<EC_SigResult> ok = groups.entrySet().stream().filter((e) -> e.getKey().equals("ok")).findFirst().get().getValue();
-        for (EC_SigResult sig : ok) {
-            ecdsaTest(sig, Result.ExpectedValue.SUCCESS, null);
+        for (int i = 0; i < cfg.number; ++i) {
+            Collections.shuffle(ok);
+            for (EC_SigResult sig : ok) {
+                ecdsaTest(sig, Result.ExpectedValue.SUCCESS, null);
+            }
         }
     }
 
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java
index 6679f60..19b9ddd 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java
@@ -14,6 +14,7 @@ import cz.crcs.ectester.reader.CardMngr;
 import cz.crcs.ectester.reader.ECTesterReader;
 import cz.crcs.ectester.reader.command.Command;
 
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -47,8 +48,11 @@ public class CardTwistSuite extends CardTestSuite {
                 Test objectEcdh = CompoundTest.any(Result.ExpectedValue.SUCCESS, CardUtil.getKATypeString(EC_Consts.KeyAgreement_ALG_EC_SVDP_DH) + " test with twist pubkey.", setPub, ecdh);
                 Command ecdhCommand = new Command.ECDH_direct(this.card, CardConsts.KEYPAIR_LOCAL, CardConsts.EXPORT_FALSE, EC_Consts.TRANSFORMATION_NONE, EC_Consts.KeyAgreement_ALG_EC_SVDP_DH, pub.flatten());
                 Test rawEcdh = CommandTest.expect(ecdhCommand, Result.ExpectedValue.FAILURE, "Card correctly rejected point on twist.", "Card incorrectly accepted point on twist.");
-                ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " twist key test.", objectEcdh, rawEcdh));
+                for (int i = 0; i < cfg.number; ++i) {
+                    ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " twist key test.", objectEcdh, rawEcdh));
+                }
             }
+            Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Perform ECDH with public points on twist.", ecdhTests.toArray(new Test[0]));
 
             Test tests = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Do tests.", ecdh);