Test with default curve if possible.

Final changes to the new test suites and scripts. (#16)

* Final changes in some of the test suites.

* Final changes to testing scripts.

* Roll back recent changes in the cofactor test suite.

* Change key generation method in Degenerate, Invalid, and Twist test suites.

* Small cosmetic change to the edge-cases test suite.

* Small change to run_test_suite.sh

8 files changed, 75 insertions, 30 deletions

diff --git a/src/cz/crcs/ectester/common/util/ECUtil.java b/src/cz/crcs/ectester/common/util/ECUtil.java
index ccc6c48..e047439 100644
--- a/src/cz/crcs/ectester/common/util/ECUtil.java
+++ b/src/cz/crcs/ectester/common/util/ECUtil.java
@@ -461,5 +461,13 @@ public class ECUtil {
         return null;
     }
 
-
+    public static boolean equalKeyPairParameters(ECPrivateKey priv, ECPublicKey pub) {
+        if(priv == null || pub == null) {
+            return false;
+        }
+        return priv.getParams().getCurve().equals(pub.getParams().getCurve()) &&
+                priv.getParams().getCofactor() == pub.getParams().getCofactor() &&
+                priv.getParams().getGenerator().equals(pub.getParams().getGenerator()) &&
+                priv.getParams().getOrder().equals(pub.getParams().getOrder());
+    }
 }
diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java
index b7e6f33..9ab8a39 100644
--- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java
+++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java
@@ -76,15 +76,25 @@ public class StandaloneDegenerateSuite extends StandaloneTestSuite {
             ECParameterSpec spec = curve.toSpec();
             KeyGeneratorTestable kgt = new KeyGeneratorTestable(kpg, spec);
 
+            Test generateSuccess;
             Test generate =  KeyGeneratorTest.expectError(kgt, Result.ExpectedValue.ANY);
             runTest(generate);
             KeyPair kp = kgt.getKeyPair();
-            if(kp == null) {
-                Test generateFail = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generating KeyPair has failed on " + curve.getId() + ". " + "KeyAgreement tests will be skipped.", generate);
-                doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Degenerate curve test of " + curve.getId() + ".", generateFail));
-                continue;
+            if(kp != null) {
+                generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generate);
+            } else { //If KeyPair generation fails, try generating it on a default curve instead. Use this key only if it has the same domain parameters as our public key.
+                KeyGeneratorTestable kgtOnDefaultCurve = new KeyGeneratorTestable(kpg, curve.getBits());
+                Test generateOnDefaultCurve = KeyGeneratorTest.expectError(kgtOnDefaultCurve, Result.ExpectedValue.ANY);
+                runTest(generateOnDefaultCurve);
+                kp = kgtOnDefaultCurve.getKeyPair();
+                if(kp != null && ECUtil.equalKeyPairParameters((ECPrivateKey) kp.getPrivate(), ECUtil.toPublicKey(keys.get(0)))) {
+                    generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generateOnDefaultCurve);
+                } else {
+                    Test generateFail = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generating KeyPair has failed on " + curve.getId() + ". " + "KeyAgreement tests will be skipped.", generate);
+                    doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Degenerate curve test of " + curve.getId() + ".", generateFail));
+                    continue;
+                }
             }
-            Test generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generate);
             ECPrivateKey ecpriv = (ECPrivateKey) kp.getPrivate();
 
             List<Test> allKaTests = new LinkedList<>();
diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java
index f84bedc..3624aaa 100644
--- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java
+++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java
@@ -271,7 +271,7 @@ public class StandaloneEdgeCasesSuite extends StandaloneTestSuite {
         if(kp == null) {
             Test generateFail = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generating KeyPair has failed on "
                     + secp160r1.getBits() + "b secp160r1." + " Other tests will be skipped.", generate);
-            doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test private key values near zero, near p and near/larger than the order." + secp160r1.getId() + ".", generateFail));
+            doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test private key values near zero, near p and near/larger than the order on" + secp160r1.getId() + ".", generateFail));
             return;
         }
         Test generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate KeyPair.", generate);
diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java
index 84c3085..ace8945 100644
--- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java
+++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java
@@ -75,15 +75,25 @@ public class StandaloneInvalidSuite extends StandaloneTestSuite {
             ECParameterSpec spec = curve.toSpec();
             KeyGeneratorTestable kgt = new KeyGeneratorTestable(kpg, spec);
 
+            Test generateSuccess;
             Test generate =  KeyGeneratorTest.expectError(kgt, Result.ExpectedValue.ANY);
             runTest(generate);
             KeyPair kp = kgt.getKeyPair();
-            if(kp == null) {
-                Test generateFail = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generating KeyPair has failed on " + curve.getId() + ". " + "KeyAgreement tests will be skipped.", generate);
-                doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Invalid curve test of " + curve.getId() + ".", generateFail));
-                continue;
+            if(kp != null) {
+                generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generate);
+            } else { //If KeyPair generation fails, try generating it on a default curve instead. Use this key only if it has the same domain parameters as our public key.
+                KeyGeneratorTestable kgtOnDefaultCurve = new KeyGeneratorTestable(kpg, curve.getBits());
+                Test generateOnDefaultCurve = KeyGeneratorTest.expectError(kgtOnDefaultCurve, Result.ExpectedValue.ANY);
+                runTest(generateOnDefaultCurve);
+                kp = kgtOnDefaultCurve.getKeyPair();
+                if(kp != null && ECUtil.equalKeyPairParameters((ECPrivateKey) kp.getPrivate(), ECUtil.toPublicKey(keys.get(0)))) {
+                    generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generateOnDefaultCurve);
+                } else {
+                    Test generateFail = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generating KeyPair has failed on " + curve.getId() + ". " + "KeyAgreement tests will be skipped.", generate);
+                    doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Invalid curve test of " + curve.getId() + ".", generateFail));
+                    continue;
+                }
             }
-            Test generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generate);
             ECPrivateKey ecpriv = (ECPrivateKey) kp.getPrivate();
 
             List<Test> allKaTests = new LinkedList<>();
diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneTestVectorSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneTestVectorSuite.java
index 2b43464..1e1889c 100644
--- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneTestVectorSuite.java
+++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneTestVectorSuite.java
@@ -3,6 +3,8 @@ package cz.crcs.ectester.standalone.test.suites;
 import cz.crcs.ectester.common.cli.TreeCommandLine;
 import cz.crcs.ectester.common.ec.*;
 import cz.crcs.ectester.common.output.TestWriter;
+import cz.crcs.ectester.common.test.CompoundTest;
+import cz.crcs.ectester.common.test.Result;
 import cz.crcs.ectester.common.util.ECUtil;
 import cz.crcs.ectester.data.EC_Store;
 import cz.crcs.ectester.standalone.ECTesterStandalone;
@@ -29,7 +31,6 @@ public class StandaloneTestVectorSuite extends StandaloneTestSuite {
     protected void runTests() throws Exception {
         Map<String, EC_KAResult> results = EC_Store.getInstance().getObjects(EC_KAResult.class, "test");
         for (EC_KAResult result : results.values()) {
-
             if(!"DH_PLAIN".equals(result.getKA())) {
                 continue;
             }
@@ -56,7 +57,7 @@ public class StandaloneTestVectorSuite extends StandaloneTestSuite {
             KeyAgreementIdent kaIdent = KeyAgreementIdent.get("ECDH");
             KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider());
             KeyAgreementTestable testable = new KeyAgreementTestable(ka, privkey, pubkey);
-            doTest(KeyAgreementTest.match(testable, result.getData(0)));
+            doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test vector " + result.getId(), KeyAgreementTest.match(testable, result.getData(0))));
         }
     }
 }
diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java
index 2b72ce6..f182952 100644
--- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java
+++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java
@@ -75,15 +75,25 @@ public class StandaloneTwistSuite extends StandaloneTestSuite {
             ECParameterSpec spec = curve.toSpec();
             KeyGeneratorTestable kgt = new KeyGeneratorTestable(kpg, spec);
 
+            Test generateSuccess;
             Test generate =  KeyGeneratorTest.expectError(kgt, Result.ExpectedValue.ANY);
             runTest(generate);
             KeyPair kp = kgt.getKeyPair();
-            if(kp == null) {
-                Test generateFail = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generating KeyPair has failed on " + curve.getId() + ". " + "KeyAgreement tests will be skipped.", generate);
-                doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Twist test of " + curve.getId() + ".", generateFail));
-                continue;
+            if(kp != null) {
+                generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generate);
+            } else { //If KeyPair generation fails, try generating it on a default curve instead. Use this key only if it has the same domain parameters as our public key.
+                KeyGeneratorTestable kgtOnDefaultCurve = new KeyGeneratorTestable(kpg, curve.getBits());
+                Test generateOnDefaultCurve = KeyGeneratorTest.expectError(kgtOnDefaultCurve, Result.ExpectedValue.ANY);
+                runTest(generateOnDefaultCurve);
+                kp = kgtOnDefaultCurve.getKeyPair();
+                if(kp != null && ECUtil.equalKeyPairParameters((ECPrivateKey) kp.getPrivate(), ECUtil.toPublicKey(keys.get(0)))) {
+                    generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generateOnDefaultCurve);
+                } else {
+                    Test generateFail = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generating KeyPair has failed on " + curve.getId() + ". " + "KeyAgreement tests will be skipped.", generate);
+                    doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Twist test of " + curve.getId() + ".", generateFail));
+                    continue;
+                }
             }
-            Test generateSuccess = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Generate keypair.", generate);
             ECPrivateKey ecpriv = (ECPrivateKey) kp.getPrivate();
 
             List<Test> allKaTests = new LinkedList<>();
diff --git a/util/run_all_suites.sh b/util/run_all_suites.sh
index dd4dc7e..c861efa 100755
--- a/util/run_all_suites.sh
+++ b/util/run_all_suites.sh
@@ -15,8 +15,7 @@ mkdir $tempfolder
 while read -r suite; do
     echo "**Run $suite suite on all the libraries:"
     bash run_test_suite.sh $suite
-    mkdir $tempfolder/$suite
-    unzip results_$suite.zip -d $tempfolder/$suite
+    unzip results_$suite.zip -d $tempfolder
     rm results_$suite.zip
 done <<< "$suites"
 
diff --git a/util/run_test_suite.sh b/util/run_test_suite.sh
index c465c79..67457c1 100755
--- a/util/run_test_suite.sh
+++ b/util/run_test_suite.sh
@@ -4,7 +4,8 @@
 # runs the specified suite on all installed libraries
 #
 suite=${1,,}
-tempfolder=.temp_results
+extra_args="" #e.g., -kt ECDH -st ECDSA
+tempfolder="temp_results"
 cur=$PWD
 timeout=10
 
@@ -23,9 +24,13 @@ mkdir $tempfolder
 run="$(which java) -jar ECTesterStandalone-dist.jar"
 libs=$($run list-libs | grep -P "^\t-" | cut -d"-" -f 2 | cut -d"(" -f1)
 while read -r lib; do
-    echo "Testing library: $lib..."
-    filename=$tempfolder/$"${lib// /_}"-${suite}_suite-results.txt
+    if [[ $lib == *"BoringSSL"* ]]; then
+        lib=BoringSSL
+    fi
+    mkdir -p $tempfolder/${suite}/$"${lib// /_}"
+    filename=$tempfolder/${suite}/$"${lib// /_}"/results.txt
 
+    echo "Testing library: $lib..."
     #Botan and Crypto++ don't recognize default kgt type EC, specify kgt=ECDH instead.
     if [[ $lib == *"Botan"* ]] || [[ $lib == *"Crypto++"* ]]; then
         args="-gt ECDH"
@@ -35,16 +40,16 @@ while read -r lib; do
 
     #Wrong suite can cause a freeze in some libraries. Try running the tests again with the -skip argument if it happens. Default timeout is 10s.
     if [[ $suite == "wrong" ]]; then
-        timeout ${timeout}s $run test $args $suite "$lib" > $filename 2>&1
+        timeout ${timeout}s $run test $args $extra_args $suite "$lib" > $filename 2>&1
         if [[ $? -eq 124 ]]; then
                 echo "#" >> $filename
                 echo "# NOTE: Tests timeouted at this point after taking longer than ${timeout}s. What follows next is a second run with -skip argument." >> $filename
                 echo "#" >> $filename
-                $run test $args $suite -skip "$lib" >> $filename 2>&1
+                $run test $args $extra_args $suite -skip "$lib" >> $filename 2>&1
         fi
     #Composite suite can also cause a freeze, but this time there is no -skip argument.
     elif [[ $suite == "composite" ]]; then
-        timeout ${timeout}s $run test $args $suite "$lib" > $filename 2>&1
+        timeout ${timeout}s $run test $args $extra_args $suite "$lib" > $filename 2>&1
         if [[ $? -eq 124 ]]; then
                 echo "#" >> $filename
                 echo "# NOTE: Tests timeouted at this point after taking longer than ${timeout}s." >> $filename
@@ -52,9 +57,9 @@ while read -r lib; do
         fi
     #Signature suite requires SHA1withECDSA signature type
     elif [[ $suite == "signature" ]]; then
-        $run test $args -st SHA1withECDSA $suite "$lib" > $tempfolder/$"${lib// /_}"-${suite}_suite-results.txt 2>&1
+        $run test $args $extra_args -st SHA1withECDSA $suite "$lib" > $filename 2>&1
     else
-        $run test $args $suite "$lib" > $tempfolder/$"${lib// /_}"-${suite}_suite-results.txt 2>&1
+        $run test $args $extra_args $suite "$lib" > $filename 2>&1
     fi
 done <<< "$libs"
 
@@ -67,7 +72,9 @@ if [[ -f $cur/results_$suite.zip ]]; then
     rm -f $cur/results_$suite.zip
 fi
 echo 'Creating archive...'
-zip -r -j $cur/results_$suite.zip $tempfolder/
+cd $tempfolder
+zip -r $cur/results_$suite.zip .
+cd ..
 rm -rf $tempfolder
 
 echo "Finished. The results can be found in results_$suite.zip."