diff options
9 files changed, 123 insertions, 21 deletions
diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneCofactorSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneCofactorSuite.java index b4f6f30..52b0fbf 100644 --- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneCofactorSuite.java +++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneCofactorSuite.java @@ -90,11 +90,10 @@ public class StandaloneCofactorSuite extends StandaloneTestSuite { List<Test> allKaTests = new LinkedList<>(); for (KeyAgreementIdent kaIdent : cfg.selected.getKAs()) { if (kaAlgo == null || kaIdent.containsAny(kaTypes)) { - KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); - List<Test> specificKaTests = new LinkedList<>(); for (EC_Key.Public pub : keys) { ECPublicKey ecpub = ECUtil.toPublicKey(pub); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, ecpriv, ecpub); Test keyAgreement = KeyAgreementTest.expectError(testable, Result.ExpectedValue.FAILURE); specificKaTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " cofactor key test.", keyAgreement)); diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneCompositeSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneCompositeSuite.java index bc843e6..c59d864 100644 --- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneCompositeSuite.java +++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneCompositeSuite.java @@ -100,11 +100,10 @@ public class StandaloneCompositeSuite extends StandaloneTestSuite { List<Test> allKaTests = new LinkedList<>(); for (KeyAgreementIdent kaIdent : cfg.selected.getKAs()) { if (kaAlgo == null || kaIdent.containsAny(kaTypes)) { - KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); - List<Test> specificKaTests = new LinkedList<>(); for (EC_Key.Public pub : curveKeys.getValue()) { ECPublicKey ecpub = ECUtil.toPublicKey(pub); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, ecpriv ,ecpub); Test keyAgreement = KeyAgreementTest.expectError(testable, Result.ExpectedValue.FAILURE); specificKaTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Composite test of " + curve.getId() + ", with generated private key, " + pub.getDesc(), keyAgreement)); @@ -122,7 +121,7 @@ public class StandaloneCompositeSuite extends StandaloneTestSuite { Map<String, EC_Curve> results = EC_Store.getInstance().getObjects(EC_Curve.class, "composite"); Map<String, List<EC_Curve>> groups = EC_Store.mapToPrefix(results.values()); - /* Test the whole curves with both keypairs generated on card(no small-order public points provided). + /* Test the whole curves with both keypairs generated by the library(no small-order public points provided). */ List<EC_Curve> wholeCurves = groups.entrySet().stream().filter((e) -> e.getKey().equals("whole")).findFirst().get().getValue(); testGroup(wholeCurves, kpg, "Composite generator order", Result.ExpectedValue.FAILURE); @@ -132,7 +131,7 @@ public class StandaloneCompositeSuite extends StandaloneTestSuite { List<EC_Curve> smallRCurves = groups.entrySet().stream().filter((e) -> e.getKey().equals("small")).findFirst().get().getValue(); testGroup(smallRCurves, kpg, "Small generator order", Result.ExpectedValue.FAILURE); - /* Test increasingly larger prime R, to determine where/if card behavior changes. + /* Test increasingly larger prime R, to determine where/if the behavior changes. */ List<EC_Curve> varyingCurves = groups.entrySet().stream().filter((e) -> e.getKey().equals("varying")).findFirst().get().getValue(); testGroup(varyingCurves, kpg, null, Result.ExpectedValue.ANY); diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java index e24d4b6..b7e6f33 100644 --- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java +++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneDegenerateSuite.java @@ -90,11 +90,10 @@ public class StandaloneDegenerateSuite extends StandaloneTestSuite { List<Test> allKaTests = new LinkedList<>(); for (KeyAgreementIdent kaIdent : cfg.selected.getKAs()) { if (kaAlgo == null || kaIdent.containsAny(kaTypes)) { - KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); - List<Test> specificKaTests = new LinkedList<>(); for (EC_Key.Public pub : keys) { ECPublicKey ecpub = ECUtil.toPublicKey(pub); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, ecpriv, ecpub); Test keyAgreement = KeyAgreementTest.expectError(testable, Result.ExpectedValue.FAILURE); specificKaTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " degenerate key test.", keyAgreement)); diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java index 6983e98..f84bedc 100644 --- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java +++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneEdgeCasesSuite.java @@ -24,6 +24,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.security.KeyPair; import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; import java.security.interfaces.ECPrivateKey; import java.security.interfaces.ECPublicKey; import java.security.spec.ECParameterSpec; @@ -34,7 +35,7 @@ import java.util.stream.Collectors; * @author David Hofman */ public class StandaloneEdgeCasesSuite extends StandaloneTestSuite { - KeyAgreement ka; + KeyAgreementIdent kaIdent; public StandaloneEdgeCasesSuite(TestWriter writer, ECTesterStandalone.Config cfg, TreeCommandLine cli) { super(writer, cfg, cli, "edge-cases", "The edge-cases test suite tests various inputs to ECDH which may cause an implementation to achieve a certain edge-case state during it.", @@ -51,7 +52,6 @@ public class StandaloneEdgeCasesSuite extends StandaloneTestSuite { String kaAlgo = cli.getOptionValue("test.ka-type"); String kpgAlgo = cli.getOptionValue("test.kpg-type"); - KeyAgreementIdent kaIdent; if (kaAlgo == null) { // try ECDH, if not, fail with: need to specify ka algo. Optional<KeyAgreementIdent> kaIdentOpt = cfg.selected.getKAs().stream() @@ -75,7 +75,6 @@ public class StandaloneEdgeCasesSuite extends StandaloneTestSuite { return; } } - ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyPairGeneratorIdent kpgIdent; if (kpgAlgo == null) { @@ -133,6 +132,7 @@ public class StandaloneEdgeCasesSuite extends StandaloneTestSuite { ECPrivateKey ecpriv = ECUtil.toPrivateKey(EC_Store.getInstance().getObject(EC_Key.Private.class, privkeyId)); ECPublicKey ecpub = ECUtil.toPublicKey(EC_Store.getInstance().getObject(EC_Key.Public.class, pubkeyId)); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, ecpriv, ecpub); Test ecdh = KeyAgreementTest.match(testable, value.getData(0)); Test one = CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test " + id + ".", ecdh); @@ -147,12 +147,14 @@ public class StandaloneEdgeCasesSuite extends StandaloneTestSuite { EC_KAResult openssl_bug = EC_Store.getInstance().getObject(EC_KAResult.class, "misc", "openssl-bug"); ECPrivateKey ecpriv = ECUtil.toPrivateKey(EC_Store.getInstance().getObject(EC_Key.Private.class, openssl_bug.getOtherKey())); ECPublicKey ecpub = ECUtil.toPublicKey(EC_Store.getInstance().getObject(EC_Key.Public.class, openssl_bug.getOneKey())); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, ecpriv, ecpub); Test ecdh = KeyAgreementTest.function(testable, new TestCallback<KeyAgreementTestable>() { @Override public Result apply(KeyAgreementTestable testable) { - if (!testable.ok()) + if (!testable.ok()) { return new Result(Result.Value.FAILURE, "ECDH was unsuccessful."); + } if (ByteUtil.compareBytes(testable.getSecret(), 0, openssl_bug.getData(0), 0, testable.getSecret().length)) { return new Result(Result.Value.FAILURE, "OpenSSL bug is present, derived secret matches example."); } @@ -303,8 +305,9 @@ public class StandaloneEdgeCasesSuite extends StandaloneTestSuite { doTest(CompoundTest.all(Result.ExpectedValue.SUCCESS, "Test private key values near zero, near p and near/larger than the order.", generateSuccess, zeroTest, pTest, rTest)); } - private Test ecdhTest(ECPublicKey pub, BigInteger SParam, ECParameterSpec spec, String desc, Result.ExpectedValue expect) { + private Test ecdhTest(ECPublicKey pub, BigInteger SParam, ECParameterSpec spec, String desc, Result.ExpectedValue expect) throws NoSuchAlgorithmException { ECPrivateKey priv = new RawECPrivateKey(SParam, spec); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, priv, pub); return CompoundTest.all(Result.ExpectedValue.SUCCESS, desc, KeyAgreementTest.expectError(testable, expect)); } diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java index 310e815..84c3085 100644 --- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java +++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneInvalidSuite.java @@ -89,11 +89,10 @@ public class StandaloneInvalidSuite extends StandaloneTestSuite { List<Test> allKaTests = new LinkedList<>(); for (KeyAgreementIdent kaIdent : cfg.selected.getKAs()) { if (kaAlgo == null || kaIdent.containsAny(kaTypes)) { - KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); - List<Test> specificKaTests = new LinkedList<>(); for (EC_Key.Public pub : keys) { ECPublicKey ecpub = ECUtil.toPublicKey(pub); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, ecpriv, ecpub); Test keyAgreement = KeyAgreementTest.expectError(testable, Result.ExpectedValue.FAILURE); specificKaTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " invalid key test.", keyAgreement)); diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java index b903153..2b72ce6 100644 --- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java +++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneTwistSuite.java @@ -89,11 +89,10 @@ public class StandaloneTwistSuite extends StandaloneTestSuite { List<Test> allKaTests = new LinkedList<>(); for (KeyAgreementIdent kaIdent : cfg.selected.getKAs()) { if (kaAlgo == null || kaIdent.containsAny(kaTypes)) { - KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); - List<Test> specificKaTests = new LinkedList<>(); for (EC_Key.Public pub : keys) { ECPublicKey ecpub = ECUtil.toPublicKey(pub); + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, ecpriv, ecpub); Test keyAgreement = KeyAgreementTest.expectError(testable, Result.ExpectedValue.FAILURE); specificKaTests.add(CompoundTest.all(Result.ExpectedValue.SUCCESS, pub.getId() + " twist key test.", keyAgreement)); diff --git a/src/cz/crcs/ectester/standalone/test/suites/StandaloneWrongSuite.java b/src/cz/crcs/ectester/standalone/test/suites/StandaloneWrongSuite.java index 589c0d7..79b0b7d 100644 --- a/src/cz/crcs/ectester/standalone/test/suites/StandaloneWrongSuite.java +++ b/src/cz/crcs/ectester/standalone/test/suites/StandaloneWrongSuite.java @@ -22,6 +22,7 @@ import javax.crypto.KeyAgreement; import java.math.BigInteger; import java.security.KeyPair; import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; import java.security.interfaces.ECPrivateKey; import java.security.interfaces.ECPublicKey; import java.security.spec.*; @@ -32,7 +33,7 @@ import java.util.stream.Collectors; * @author David Hofman */ public class StandaloneWrongSuite extends StandaloneTestSuite { - private KeyAgreement ka; + private KeyAgreementIdent kaIdent; private KeyPairGenerator kpg; public StandaloneWrongSuite(TestWriter writer, ECTesterStandalone.Config cfg, TreeCommandLine cli) { @@ -76,7 +77,6 @@ public class StandaloneWrongSuite extends StandaloneTestSuite { } kpg = kpgIdent.getInstance(cfg.selected.getProvider()); - KeyAgreementIdent kaIdent; if (kaAlgo == null) { // try ECDH, if not, fail with: need to specify ka algo. Optional<KeyAgreementIdent> kaIdentOpt = cfg.selected.getKAs().stream() @@ -100,7 +100,6 @@ public class StandaloneWrongSuite extends StandaloneTestSuite { return; } } - ka = kaIdent.getInstance(cfg.selected.getProvider()); /* Just do the default run on the wrong curves. * These should generally fail, the curves aren't curves. @@ -283,7 +282,7 @@ public class StandaloneWrongSuite extends StandaloneTestSuite { } } - private Test ecdhTest(ECParameterSpec spec, String desc) { + private Test ecdhTest(ECParameterSpec spec, String desc) throws NoSuchAlgorithmException { //generate KeyPair KeyGeneratorTestable kgt = new KeyGeneratorTestable(kpg, spec); Test generate = KeyGeneratorTest.expectError(kgt, Result.ExpectedValue.FAILURE); @@ -296,6 +295,7 @@ public class StandaloneWrongSuite extends StandaloneTestSuite { ECPrivateKey priv = (ECPrivateKey) kp.getPrivate(); //perform ECDH + KeyAgreement ka = kaIdent.getInstance(cfg.selected.getProvider()); KeyAgreementTestable testable = new KeyAgreementTestable(ka, priv, pub); Test ecdh = KeyAgreementTest.expect(testable, Result.ExpectedValue.FAILURE); return CompoundTest.all(Result.ExpectedValue.SUCCESS, desc, generate, ecdh); diff --git a/util/run_all_suites.sh b/util/run_all_suites.sh new file mode 100755 index 0000000..dd4dc7e --- /dev/null +++ b/util/run_all_suites.sh @@ -0,0 +1,30 @@ +#!/usr/bin/bash +# +# ECTesterStandalone testing script, +# runs all the suites on all the libraries +# +tempfolder=.temp_results +cur=$PWD +cd "$(dirname "${BASH_SOURCE[0]}")"/../dist +run="$(which java) -jar ECTesterStandalone-dist.jar" +suites=$($run list-suites | grep -P "^ -" | cut -c3-) +cd $cur + +rm -rf $tempfolder +mkdir $tempfolder +while read -r suite; do + echo "**Run $suite suite on all the libraries:" + bash run_test_suite.sh $suite + mkdir $tempfolder/$suite + unzip results_$suite.zip -d $tempfolder/$suite + rm results_$suite.zip +done <<< "$suites" + +if [[ -f results_all.zip ]]; then + echo '**Removing old archive...' + rm -f results_all.zip +fi +echo '**Creating archive...' +cd $tempfolder && zip -r ../results_all.zip . && cd .. +rm -rf $tempfolder +echo "**All tests finished! The results can be found in results_all.zip" diff --git a/util/run_test_suite.sh b/util/run_test_suite.sh new file mode 100755 index 0000000..c465c79 --- /dev/null +++ b/util/run_test_suite.sh @@ -0,0 +1,74 @@ +#!/usr/bin/bash +# +# ECTesterStandalone testing script, +# runs the specified suite on all installed libraries +# +suite=${1,,} +tempfolder=.temp_results +cur=$PWD +timeout=10 + +cd "$(dirname "${BASH_SOURCE[0]}")"/../dist +if [[ $# -eq 0 ]]; then + echo 'No test suite specified.' + exit 0 +fi +if [[ ! -f ECTesterStandalone-dist.jar ]]; then + echo 'ECTesterStandalone-dist.jar not found. Build ECTesterStandalone first.' + exit 0 +fi + +rm -rf $tempfolder +mkdir $tempfolder +run="$(which java) -jar ECTesterStandalone-dist.jar" +libs=$($run list-libs | grep -P "^\t-" | cut -d"-" -f 2 | cut -d"(" -f1) +while read -r lib; do + echo "Testing library: $lib..." + filename=$tempfolder/$"${lib// /_}"-${suite}_suite-results.txt + + #Botan and Crypto++ don't recognize default kgt type EC, specify kgt=ECDH instead. + if [[ $lib == *"Botan"* ]] || [[ $lib == *"Crypto++"* ]]; then + args="-gt ECDH" + else + args="" + fi + + #Wrong suite can cause a freeze in some libraries. Try running the tests again with the -skip argument if it happens. Default timeout is 10s. + if [[ $suite == "wrong" ]]; then + timeout ${timeout}s $run test $args $suite "$lib" > $filename 2>&1 + if [[ $? -eq 124 ]]; then + echo "#" >> $filename + echo "# NOTE: Tests timeouted at this point after taking longer than ${timeout}s. What follows next is a second run with -skip argument." >> $filename + echo "#" >> $filename + $run test $args $suite -skip "$lib" >> $filename 2>&1 + fi + #Composite suite can also cause a freeze, but this time there is no -skip argument. + elif [[ $suite == "composite" ]]; then + timeout ${timeout}s $run test $args $suite "$lib" > $filename 2>&1 + if [[ $? -eq 124 ]]; then + echo "#" >> $filename + echo "# NOTE: Tests timeouted at this point after taking longer than ${timeout}s." >> $filename + echo "#" >> $filename + fi + #Signature suite requires SHA1withECDSA signature type + elif [[ $suite == "signature" ]]; then + $run test $args -st SHA1withECDSA $suite "$lib" > $tempfolder/$"${lib// /_}"-${suite}_suite-results.txt 2>&1 + else + $run test $args $suite "$lib" > $tempfolder/$"${lib// /_}"-${suite}_suite-results.txt 2>&1 + fi +done <<< "$libs" + +#Comment out these two lines to keep java error logs. They are removed by default to prevent unnecessary cluttering of dist folder. +echo 'Removing java error logs...' +find . -type f -name 'hs_err_*' -exec rm {} \; + +if [[ -f $cur/results_$suite.zip ]]; then + echo 'Removing old archive...' + rm -f $cur/results_$suite.zip +fi +echo 'Creating archive...' +zip -r -j $cur/results_$suite.zip $tempfolder/ +rm -rf $tempfolder + +echo "Finished. The results can be found in results_$suite.zip." +exit 1 |