Add ability to skip tests/run only a part of a test suite.

12 files changed, 50 insertions, 22 deletions

diff --git a/src/cz/crcs/ectester/reader/CardMngr.java b/src/cz/crcs/ectester/reader/CardMngr.java
index 637be56..921a9c8 100644
--- a/src/cz/crcs/ectester/reader/CardMngr.java
+++ b/src/cz/crcs/ectester/reader/CardMngr.java
@@ -1,6 +1,5 @@
 package cz.crcs.ectester.reader;
 
-import com.licel.jcardsim.io.CAD;
 import com.licel.jcardsim.io.JavaxSmartCardInterface;
 import cz.crcs.ectester.common.util.ByteUtil;
 import javacard.framework.AID;
diff --git a/src/cz/crcs/ectester/reader/ECTesterReader.java b/src/cz/crcs/ectester/reader/ECTesterReader.java
index 89cfca1..2b78cb0 100644
--- a/src/cz/crcs/ectester/reader/ECTesterReader.java
+++ b/src/cz/crcs/ectester/reader/ECTesterReader.java
@@ -272,8 +272,8 @@ public class ECTesterReader {
         actions.addOption(Option.builder("h").longOpt("help").desc("Print help.").build());
         actions.addOption(Option.builder("ln").longOpt("list-named").desc("Print the list of supported named curves and keys.").hasArg().argName("what").optionalArg(true).build());
         actions.addOption(Option.builder("e").longOpt("export").desc("Export the defaut curve parameters of the card(if any).").build());
-        actions.addOption(Option.builder("g").longOpt("generate").desc("Generate [amount] of EC keys.").hasArg().argName("amount").optionalArg(true).build());
-        actions.addOption(Option.builder("t").longOpt("test").desc("Test ECC support. [test_suite]:\n- default:\n- compression:\n- invalid:\n- twist:\n- degenerate:\n- cofactor:\n- wrong:\n- composite:\n- test-vectors:\n- edge-cases:\n- miscellaneous:").hasArg().argName("test_suite").optionalArg(true).build());
+        actions.addOption(Option.builder("g").longOpt("generate").desc("Generate <amount> of EC keys.").hasArg().argName("amount").optionalArg(true).build());
+        actions.addOption(Option.builder("t").longOpt("test").desc("Test ECC support. Optionally specify a test number to run only a part of a test suite. <test_suite>:\n- default:\n- compression:\n- invalid:\n- twist:\n- degenerate:\n- cofactor:\n- wrong:\n- composite:\n- test-vectors:\n- edge-cases:\n- miscellaneous:").hasArg().argName("test_suite[:from[:to]]").optionalArg(true).build());
         actions.addOption(Option.builder("dh").longOpt("ecdh").desc("Do EC KeyAgreement (ECDH...), [count] times.").hasArg().argName("count").optionalArg(true).build());
         actions.addOption(Option.builder("dsa").longOpt("ecdsa").desc("Sign data with ECDSA, [count] times.").hasArg().argName("count").optionalArg(true).build());
         actions.addOption(Option.builder("ls").longOpt("list-suites").desc("List supported test suites.").build());
@@ -503,7 +503,7 @@ public class ECTesterReader {
                 break;
         }
 
-        suite.run();
+        suite.run(cfg.testFrom, cfg.testTo);
     }
 
     /**
@@ -702,6 +702,8 @@ public class ECTesterReader {
         //Action-related options
         public String listNamed;
         public String testSuite;
+        public int testFrom;
+        public int testTo;
         public int generateAmount;
         public int ECKACount;
         public byte ECKAType = KeyAgreement_ALG_EC_SVDP_DH;
@@ -827,7 +829,34 @@ public class ECTesterReader {
                     primeField = true;
                 }
 
-                testSuite = cli.getOptionValue("test", "default").toLowerCase();
+                String suiteOpt = cli.getOptionValue("test", "default").toLowerCase();
+                if (suiteOpt.contains(":")) {
+                    String[] parts = suiteOpt.split(":");
+                    testSuite = parts[0];
+                    try {
+                        testFrom = Integer.parseInt(parts[1]);
+                    } catch (NumberFormatException nfe) {
+                        System.err.println("Invalid test from number: " + parts[1] + ".");
+                        return false;
+                    }
+                    if (parts.length == 3) {
+                        try {
+                            testTo = Integer.parseInt(parts[2]);
+                        } catch (NumberFormatException nfe) {
+                            System.err.println("Invalid test to number: " + parts[2] + ".");
+                            return false;
+                        }
+                    } else if (parts.length != 2) {
+                        System.err.println("Invalid test suite selection.");
+                        return false;
+                    } else {
+                        testTo = -1;
+                    }
+                } else {
+                    testSuite = suiteOpt;
+                    testFrom = 0;
+                    testTo = -1;
+                }
                 String[] tests = new String[]{"default", "composite", "compression", "invalid", "degenerate", "test-vectors", "wrong", "twist", "cofactor", "edge-cases", "miscellaneous"};
                 if (!Arrays.asList(tests).contains(testSuite)) {
                     System.err.println(Colors.error("Unknown test suite " + testSuite + ". Should be one of: " + Arrays.toString(tests)));
diff --git a/src/cz/crcs/ectester/reader/test/CardCofactorSuite.java b/src/cz/crcs/ectester/reader/test/CardCofactorSuite.java
index 2ecf4a2..762dc88 100644
--- a/src/cz/crcs/ectester/reader/test/CardCofactorSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardCofactorSuite.java
@@ -65,9 +65,9 @@ public class CardCofactorSuite extends CardTestSuite {
             Test ecdsa = CompoundTest.all(ExpectedValue.SUCCESS, "Verify random ECDSA signature by public points on non-generator subgroup.", ecdsaTests.toArray(new Test[0]));
 
             Test tests = CompoundTest.all(ExpectedValue.SUCCESS, "Perform ECDH and ECDSA tests.", ecdh, ecdsa);
+            Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), ExpectedValue.SUCCESS);
 
-            doTest(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Cofactor test of " + curve.getId() + ".", prepare, tests));
-            new Command.Cleanup(this.card).send();
+            doTest(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Cofactor test of " + curve.getId() + ".", prepare, tests, cleanup));
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardCompositeSuite.java b/src/cz/crcs/ectester/reader/test/CardCompositeSuite.java
index a693ac7..b80a0e3 100644
--- a/src/cz/crcs/ectester/reader/test/CardCompositeSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardCompositeSuite.java
@@ -96,8 +96,9 @@ public class CardCompositeSuite extends CardTestSuite {
             } else {
                 description = testName + " test of " + curve.getId() + ".";
             }
-            doTest(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, description, allocate, set, generate, ecdh));
-            new Command.Cleanup(this.card).send();
+            Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), ExpectedValue.SUCCESS);
+
+            doTest(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, description, allocate, set, generate, ecdh, cleanup));
         }
 
     }
diff --git a/src/cz/crcs/ectester/reader/test/CardCompressionSuite.java b/src/cz/crcs/ectester/reader/test/CardCompressionSuite.java
index 35cfd1d..19c452c 100644
--- a/src/cz/crcs/ectester/reader/test/CardCompressionSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardCompressionSuite.java
@@ -112,9 +112,9 @@ public class CardCompressionSuite extends CardTestSuite {
                 }
             }
             compressionTests.addAll(kaTests);
+            compressionTests.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.SUCCESS));
 
             doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Compression test of " + spec + ".", compressionTests.toArray(new Test[0])));
-            new Command.Cleanup(this.card).send();
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardDefaultSuite.java b/src/cz/crcs/ectester/reader/test/CardDefaultSuite.java
index 4480962..554003b 100644
--- a/src/cz/crcs/ectester/reader/test/CardDefaultSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardDefaultSuite.java
@@ -126,10 +126,10 @@ public class CardDefaultSuite extends CardTestSuite {
             }
             Test signTest = runTest(CompoundTest.any(ExpectedValue.SUCCESS, "Signature tests.", signTests.toArray(new Test[0])));
             supportTests.add(signTest);
+            supportTests.add(CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.SUCCESS));
 
-            ExpectedValue[] testExpects = {ExpectedValue.SUCCESS, ExpectedValue.ANY, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS};
+            ExpectedValue[] testExpects = {ExpectedValue.SUCCESS, ExpectedValue.ANY, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS, ExpectedValue.SUCCESS, ExpectedValue.ANY};
             doTest(CompoundTest.mask(testExpects, "Tests of " + keyLength + "b " + CardUtil.getKeyTypeString(field) + " support.", supportTests.toArray(new Test[0])));
-            new Command.Cleanup(this.card).send();
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardDegenerateSuite.java b/src/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
index 7483b2b..c3cf51c 100644
--- a/src/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardDegenerateSuite.java
@@ -47,9 +47,9 @@ public class CardDegenerateSuite extends CardTestSuite {
                 ecdhTests.add(CommandTest.expect(ecdhCommand, Result.ExpectedValue.FAILURE, "Card correctly rejected point on degenerate curve.", "Card incorrectly accepted point on degenerate curve."));
             }
             Test ecdh = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Perform ECDH with degenerate public points", ecdhTests.toArray(new Test[0]));
+            Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.SUCCESS);
 
-            doTest(CompoundTest.greedyAllTry(Result.ExpectedValue.SUCCESS, "Degenerate curve test of " + curve.getId(), prepare, ecdh));
-            new Command.Cleanup(this.card).send();
+            doTest(CompoundTest.greedyAllTry(Result.ExpectedValue.SUCCESS, "Degenerate curve test of " + curve.getId(), prepare, ecdh, cleanup));
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardInvalidSuite.java b/src/cz/crcs/ectester/reader/test/CardInvalidSuite.java
index 2543027..60afe75 100644
--- a/src/cz/crcs/ectester/reader/test/CardInvalidSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardInvalidSuite.java
@@ -67,9 +67,9 @@ public class CardInvalidSuite extends CardTestSuite {
             Test ecdsa = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Verify random ECDSA signature by invalid public points", ecdsaTests.toArray(new Test[0]));
 
             Test tests = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test ECDH and ECDSA with points on invalid curves.", ecdh, ecdsa);
+            Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), ExpectedValue.SUCCESS);
 
-            doTest(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Invalid curve test of " + curve.getId(), prepare, tests));
-            new Command.Cleanup(this.card).send();
+            doTest(CompoundTest.greedyAllTry(ExpectedValue.SUCCESS, "Invalid curve test of " + curve.getId(), prepare, tests, cleanup));
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardMiscSuite.java b/src/cz/crcs/ectester/reader/test/CardMiscSuite.java
index d969cf9..5dcf727 100644
--- a/src/cz/crcs/ectester/reader/test/CardMiscSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardMiscSuite.java
@@ -46,9 +46,9 @@ public class CardMiscSuite extends CardTestSuite {
             Test generate = CommandTest.expect(new Command.Generate(this.card, ECTesterApplet.KEYPAIR_BOTH), Result.ExpectedValue.SUCCESS);
             Test ka = CommandTest.expect(new Command.ECDH(this.card, ECTesterApplet.KEYPAIR_LOCAL, ECTesterApplet.KEYPAIR_REMOTE, ECTesterApplet.EXPORT_FALSE, EC_Consts.TRANSFORMATION_NONE, EC_Consts.KeyAgreement_ALG_EC_SVDP_DH), Result.ExpectedValue.SUCCESS);
             Test sig = CommandTest.expect(new Command.ECDSA(this.card, ECTesterApplet.KEYPAIR_LOCAL, EC_Consts.Signature_ALG_ECDSA_SHA, ECTesterApplet.EXPORT_FALSE, null), Result.ExpectedValue.SUCCESS);
+            Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.SUCCESS);
 
-            doTest(CompoundTest.greedyAll(Result.ExpectedValue.SUCCESS, "Tests over " + curve.getBits() + " " + catName + " curve: " + curve.getId() + ".", allocateFirst, set, generate, ka, sig));
-            new Command.Cleanup(this.card).send();
+            doTest(CompoundTest.greedyAll(Result.ExpectedValue.SUCCESS, "Tests over " + curve.getBits() + " " + catName + " curve: " + curve.getId() + ".", allocateFirst, set, generate, ka, sig, cleanup));
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardTestVectorSuite.java b/src/cz/crcs/ectester/reader/test/CardTestVectorSuite.java
index 9d39525..9a39a72 100644
--- a/src/cz/crcs/ectester/reader/test/CardTestVectorSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardTestVectorSuite.java
@@ -69,8 +69,8 @@ public class CardTestVectorSuite extends CardTestSuite {
                     return new Result(Value.SUCCESS);
                 }
             }));
+            testVector.add(CommandTest.expect(new Command.Cleanup(this.card), ExpectedValue.SUCCESS));
             doTest(CompoundTest.greedyAll(ExpectedValue.SUCCESS, "Test vector " + result.getId(), testVector.toArray(new Test[0])));
-            new Command.Cleanup(this.card).send();
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardTwistSuite.java b/src/cz/crcs/ectester/reader/test/CardTwistSuite.java
index 46da415..e7ea436 100644
--- a/src/cz/crcs/ectester/reader/test/CardTwistSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardTwistSuite.java
@@ -61,9 +61,9 @@ public class CardTwistSuite extends CardTestSuite {
             Test ecdsa = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Verify random ECDSA signature by public points on twist", ecdsaTests.toArray(new Test[0]));
 
             Test tests = CompoundTest.all(Result.ExpectedValue.SUCCESS, ecdh, ecdsa);
+            Test cleanup = CommandTest.expect(new Command.Cleanup(this.card), Result.ExpectedValue.SUCCESS);
 
-            doTest(CompoundTest.greedyAllTry(Result.ExpectedValue.SUCCESS, "Twist test of " + curve.getId(), prepare, tests));
-            new Command.Cleanup(this.card).send();
+            doTest(CompoundTest.greedyAllTry(Result.ExpectedValue.SUCCESS, "Twist test of " + curve.getId(), prepare, tests, cleanup));
         }
     }
 }
diff --git a/src/cz/crcs/ectester/reader/test/CardWrongSuite.java b/src/cz/crcs/ectester/reader/test/CardWrongSuite.java
index 6c0d5f5..2057093 100644
--- a/src/cz/crcs/ectester/reader/test/CardWrongSuite.java
+++ b/src/cz/crcs/ectester/reader/test/CardWrongSuite.java
@@ -129,7 +129,6 @@ public class CardWrongSuite extends CardTestSuite {
 
             Test wrongR = CompoundTest.all(ExpectedValue.SUCCESS, "Tests with corrupted R parameter.", primeWrongR, nonprimeWrongR);
 
-
             doTest(CompoundTest.all(ExpectedValue.SUCCESS, "Tests of " + keyLength + "b " + CardUtil.getKeyTypeString(KeyPair.ALG_EC_FP), setup, wrongPrime, resetSetup, wrongG, resetSetup.clone(), wrongR, resetSetup.clone()));
         }