8 files changed, 31 insertions, 13 deletions

diff --git a/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java b/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java
index 082be23..ddd733b 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/ECTesterReader.java
@@ -341,9 +341,10 @@ public class ECTesterReader {
         opts.addOption(Option.builder().longOpt("time").desc("Output better timing values, by running command in dry run mode and normal mode, and subtracting the two.").build());
         opts.addOption(Option.builder().longOpt("time-unit").desc("Use given time unit in measurement, one of: milli, micro, nano.").hasArg().argName("unit").build());
         opts.addOption(Option.builder().longOpt("cleanup").desc("Send the cleanup command triggering JCSystem.requestObjectDeletion() after some operations.").build());
-        opts.addOption(Option.builder("n").longOpt("number").desc("Number of repeats during testing.").hasArg().argName("number").build());
         opts.addOption(Option.builder("s").longOpt("simulate").desc("Simulate a card with jcardsim instead of using a terminal.").build());
         opts.addOption(Option.builder("y").longOpt("yes").desc("Accept all warnings and prompts.").build());
+        opts.addOption(Option.builder("n").longOpt("number").desc("Number of repeats during testing.").hasArg().argName("number").build());
+        opts.addOption(Option.builder("ts").longOpt("test-shuffle").desc("Shuffle the test suite before running it.").build());
         opts.addOption(Option.builder("tk").longOpt("test-key").desc("Key setup technique to use in test suites:\n- generate (default): Generate keypairs on the card.\n- deterministic: Prepare keypairs deterministically off-card.\n- random: Prepare keypairs randomly off-card.").hasArg().argName("option").build());
         opts.addOption(Option.builder("td").longOpt("test-data").desc("Data setup technique to use in test suites:\n- random (default): Prepare data randomly off-card.\n- deterministic: Prepare data deterministically off-card.").hasArg().argName("option").build());
 
@@ -853,7 +854,6 @@ public class ECTesterReader {
         public String timeUnit;
         public boolean cleanup = false;
         public boolean simulate = false;
-        public int number = 1;
         public boolean yes = false;
         public String format;
         public boolean color;
@@ -868,6 +868,8 @@ public class ECTesterReader {
         public byte ECKAType = EC_Consts.KeyAgreement_ALG_EC_SVDP_DH;
         public int ECDSACount;
         public byte ECDSAType = EC_Consts.Signature_ALG_ECDSA_SHA;
+        public int number = 1;
+        public boolean testShuffle;
         public String testKeySetup;
         public String testDataSetup;
 
@@ -916,7 +918,6 @@ public class ECTesterReader {
             time = cli.hasOption("time");
             cleanup = cli.hasOption("cleanup");
             simulate = cli.hasOption("simulate");
-            number = Integer.parseInt(cli.getOptionValue("number", "1"));
             yes = cli.hasOption("yes");
             color = cli.hasOption("color");
             Colors.enabled = color;
@@ -1000,6 +1001,9 @@ public class ECTesterReader {
                     primeField = true;
                 }
 
+                number = Integer.parseInt(cli.getOptionValue("number", "1"));
+                testShuffle = cli.hasOption("test-shuffle");
+
                 String suiteOpt = cli.getOptionValue("test", "default").toLowerCase();
                 if (suiteOpt.contains(":")) {
                     String[] parts = suiteOpt.split(":");
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java
index 1ed91db..01d278e 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCofactorSuite.java
@@ -54,7 +54,8 @@ public class CardCofactorSuite extends CardTestSuite {
                     ecdhTests.add(CompoundTest.all(ExpectedValue.SUCCESS, pub.getId() + " cofactor key test.", objectEcdh, rawEcdh));
                 }
             }
-            Collections.shuffle(ecdhTests);
+            if (cfg.testShuffle)
+                Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(ExpectedValue.SUCCESS, "Perform ECDH with public points on non-generator subgroup.", ecdhTests.toArray(new Test[0]));
 
             if (cfg.cleanup) {
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java
index 71890e1..7e442b0 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardCompositeSuite.java
@@ -12,6 +12,7 @@ import cz.crcs.ectester.reader.CardMngr;
 import cz.crcs.ectester.reader.ECTesterReader;
 import cz.crcs.ectester.reader.command.Command;
 
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -44,8 +45,12 @@ public class CardCompositeSuite extends CardTestSuite {
             for (EC_Key key : curveKeys.getValue()) {
                 Command ecdhCommand = new Command.ECDH_direct(this.card, CardConsts.KEYPAIR_LOCAL, CardConsts.EXPORT_FALSE, EC_Consts.TRANSFORMATION_NONE, EC_Consts.KeyAgreement_ALG_EC_SVDP_DH, key.flatten());
                 Test ecdh = CommandTest.expect(ecdhCommand, ExpectedValue.FAILURE, "Card correctly rejected to do ECDH over a composite order curve.", "Card incorrectly does ECDH over a composite order curve, leaks bits of private key.");
-                tests.add(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Composite test of " + curve.getId() + ", with " + key.getDesc(), ecdh));
+                for (int i = 0; i < cfg.number; ++i) {
+                    tests.add(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Composite test of " + curve.getId() + ", with " + key.getDesc(), ecdh));
+                }
             }
+            if (cfg.testShuffle)
+                Collections.shuffle(tests);
             Test ecdhTest = CompoundTest.all(ExpectedValue.SUCCESS, "Do ECDH.", tests.toArray(new Test[0]));
             doTest(CompoundTest.greedyAll(ExpectedValue.SUCCESS, "Composite test of " + curve.getId() + ".", prepare, ecdhTest));
         }
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
index aa69565..1bef698 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
@@ -55,7 +55,8 @@ public class CardDegenerateSuite extends CardTestSuite {
                 }
                 //TODO: actually get the result of ECDH here, as well as export privkey and compare to exponentiation in Fp^*.
             }
-            Collections.shuffle(ecdhTests);
+            if (cfg.testShuffle)
+                Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Perform ECDH with degenerate public points.", ecdhTests.toArray(new Test[0]));
             if (cfg.cleanup) {
                 Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY);
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java
index ed24e59..787c1c7 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardEdgeCasesSuite.java
@@ -101,7 +101,8 @@ public class CardEdgeCasesSuite extends CardTestSuite {
                         curveTests.add(one);
                     }
                 }
-                Collections.shuffle(curveTests);
+                if (cfg.testShuffle)
+                    Collections.shuffle(curveTests);
 
                 if (cfg.cleanup) {
                     curveTests.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY));
@@ -236,7 +237,8 @@ public class CardEdgeCasesSuite extends CardTestSuite {
             for (int i = 0; i < cfg.number; ++i) {
                 tests.addAll(Arrays.asList(zeroS, oneS, alternateS, alternateOtherS, fullS, smallerS, exactS, largerS, rm1S, rp1S, krS, krm1S, krp1S));
             }
-            Collections.shuffle(tests);
+            if (cfg.testShuffle)
+                Collections.shuffle(tests);
 
             if (cfg.cleanup) {
                 Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY);
@@ -312,7 +314,8 @@ public class CardEdgeCasesSuite extends CardTestSuite {
         for (int j = 0; j < cfg.number; ++j) {
             tests160.addAll(Arrays.asList(zeroTest, pTest, rTest));
         }
-        Collections.shuffle(tests160);
+        if (cfg.testShuffle)
+            Collections.shuffle(tests160);
         tests160.add(0, setup);
         if (cfg.cleanup) {
             tests160.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.ANY));
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java
index 69c1e52..2cb8e11 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardInvalidSuite.java
@@ -58,7 +58,8 @@ public class CardInvalidSuite extends CardTestSuite {
                     ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " invalid key test.", objectEcdh, rawEcdh));
                 }
             }
-            Collections.shuffle(ecdhTests);
+            if (cfg.testShuffle)
+                Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(ExpectedValue.SUCCESS, "Perform ECDH with invalid public points.", ecdhTests.toArray(new Test[0]));
 
             if (cfg.cleanup) {
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java
index 3cda557..e61be83 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardSignatureSuite.java
@@ -36,7 +36,8 @@ public class CardSignatureSuite extends CardTestSuite {
 
         byte[] data = "Some stuff that is not the actual data".getBytes();
         for (int i = 0; i < cfg.number; ++i) {
-            Collections.shuffle(nok);
+            if (cfg.testShuffle)
+                Collections.shuffle(nok);
             for (EC_SigResult sig : nok) {
                 ecdsaTest(sig, Result.ExpectedValue.FAILURE, data);
             }
@@ -44,7 +45,8 @@ public class CardSignatureSuite extends CardTestSuite {
 
         List<EC_SigResult> ok = groups.entrySet().stream().filter((e) -> e.getKey().equals("ok")).findFirst().get().getValue();
         for (int i = 0; i < cfg.number; ++i) {
-            Collections.shuffle(ok);
+            if (cfg.testShuffle)
+                Collections.shuffle(ok);
             for (EC_SigResult sig : ok) {
                 ecdsaTest(sig, Result.ExpectedValue.SUCCESS, null);
             }
diff --git a/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java b/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java
index 19b9ddd..1f7f222 100644
--- a/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java
+++ b/reader/src/main/java/cz/crcs/ectester/reader/test/CardTwistSuite.java
@@ -52,7 +52,8 @@ public class CardTwistSuite extends CardTestSuite {
                     ecdhTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " twist key test.", objectEcdh, rawEcdh));
                 }
             }
-            Collections.shuffle(ecdhTests);
+            if (cfg.testShuffle)
+                Collections.shuffle(ecdhTests);
             Test ecdh = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Perform ECDH with public points on twist.", ecdhTests.toArray(new Test[0]));
 
             Test tests = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Do tests.", ecdh);