Implement tracking and writing of exceptions in standalone tests.

9 files changed, 89 insertions, 47 deletions

diff --git a/src/cz/crcs/ectester/common/output/BaseYAMLTestWriter.java b/src/cz/crcs/ectester/common/output/BaseYAMLTestWriter.java
index c274cf2..609e46d 100644
--- a/src/cz/crcs/ectester/common/output/BaseYAMLTestWriter.java
+++ b/src/cz/crcs/ectester/common/output/BaseYAMLTestWriter.java
@@ -25,8 +25,8 @@ public abstract class BaseYAMLTestWriter implements TestWriter {
     @Override
     public void begin(TestSuite suite) {
         output.println("---");
-        testRun = new HashMap<>();
-        testSuite = new HashMap<>();
+        testRun = new LinkedHashMap<>();
+        testSuite = new LinkedHashMap<>();
         tests = new LinkedList<>();
         testSuite.put("name", suite.getName());
         testSuite.put("desc", suite.getDescription());
@@ -59,7 +59,7 @@ public abstract class BaseYAMLTestWriter implements TestWriter {
     }
 
     private Map<String, Object> resultObject(Result result) {
-        Map<String, Object> resultObject = new HashMap<>();
+        Map<String, Object> resultObject = new LinkedHashMap<>();
         resultObject.put("ok", result.ok());
         resultObject.put("value", result.getValue().name());
         resultObject.put("cause", causeObject(result.getCause()));
@@ -107,7 +107,7 @@ public abstract class BaseYAMLTestWriter implements TestWriter {
         options.setPrettyFlow(true);
         Yaml yaml = new Yaml(options);
 
-        Map<String, Object> result = new HashMap<>();
+        Map<String, Object> result = new LinkedHashMap<>();
         result.put("testRun", testRun);
         String out = yaml.dump(result);
 
diff --git a/src/cz/crcs/ectester/reader/output/YAMLTestWriter.java b/src/cz/crcs/ectester/reader/output/YAMLTestWriter.java
index 4c908cc..820521d 100644
--- a/src/cz/crcs/ectester/reader/output/YAMLTestWriter.java
+++ b/src/cz/crcs/ectester/reader/output/YAMLTestWriter.java
@@ -13,10 +13,7 @@ import cz.crcs.ectester.reader.test.CommandTestable;
 
 import javax.smartcardio.CardException;
 import java.io.PrintStream;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * @author Jan Jancar johny@neuromancer.sk
@@ -27,7 +24,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     }
 
     private Map<String, Object> commandObject(Command c) {
-        Map<String, Object> commandObj = new HashMap<>();
+        Map<String, Object> commandObj = new LinkedHashMap<>();
         if (c == null) {
             return commandObj;
         }
@@ -37,7 +34,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     }
 
     private Map<String, Object> responseObject(Response r) {
-        Map<String, Object> responseObj = new HashMap<>();
+        Map<String, Object> responseObj = new LinkedHashMap<>();
         if (r == null) {
             return responseObj;
         }
@@ -58,7 +55,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     protected Map<String, Object> testableObject(Testable t) {
         if (t instanceof CommandTestable) {
             CommandTestable cmd = (CommandTestable) t;
-            Map<String, Object> result = new HashMap<>();
+            Map<String, Object> result = new LinkedHashMap<>();
             result.put("type", "command");
             result.put("command", commandObject(cmd.getCommand()));
             result.put("response", responseObject(cmd.getResponse()));
@@ -68,7 +65,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     }
 
     private Map<String, Object> cplcObject(CardMngr card) {
-        Map<String, Object> result = new HashMap<>();
+        Map<String, Object> result = new LinkedHashMap<>();
         try {
             CardMngr.CPLC cplc = card.getCPLC();
             if (!cplc.values().isEmpty()) {
@@ -87,7 +84,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     protected Map<String, Object> deviceObject(TestSuite suite) {
         if (suite instanceof CardTestSuite) {
             CardTestSuite cardSuite = (CardTestSuite) suite;
-            Map<String, Object> result = new HashMap<>();
+            Map<String, Object> result = new LinkedHashMap<>();
             result.put("type", "card");
             result.put("ectester", ECTesterReader.VERSION);
             result.put("cplc", cplcObject(cardSuite.getCard()));
diff --git a/src/cz/crcs/ectester/standalone/output/TextTestWriter.java b/src/cz/crcs/ectester/standalone/output/TextTestWriter.java
index 691bea8..2e29b07 100644
--- a/src/cz/crcs/ectester/standalone/output/TextTestWriter.java
+++ b/src/cz/crcs/ectester/standalone/output/TextTestWriter.java
@@ -4,10 +4,9 @@ import cz.crcs.ectester.common.output.BaseTextTestWriter;
 import cz.crcs.ectester.common.test.TestSuite;
 import cz.crcs.ectester.common.test.Testable;
 import cz.crcs.ectester.standalone.ECTesterStandalone;
-import cz.crcs.ectester.standalone.test.base.*;
+import cz.crcs.ectester.standalone.test.base.StandaloneTestable;
 import cz.crcs.ectester.standalone.test.suites.StandaloneTestSuite;
 
-import javax.crypto.KeyAgreement;
 import java.io.PrintStream;
 
 /**
@@ -18,10 +17,25 @@ public class TextTestWriter extends BaseTextTestWriter {
         super(output);
     }
 
+    private String causeString(Object cause) {
+        if (cause == null) {
+            return "";
+        } else if (cause instanceof Exception) {
+            Exception ex = ((Exception) cause);
+            return " -> " + ex.getClass().getCanonicalName() + " : " + ex.getMessage();
+        } else {
+            return cause.toString();
+        }
+    }
+
     @Override
     protected String testableString(Testable t) {
         if (t instanceof StandaloneTestable) {
-            return ((StandaloneTestable)t).getStage().name();
+            StandaloneTestable<?> testable = (StandaloneTestable) t;
+            String stage = testable.getStage().name();
+            String exception = causeString(testable.getException());
+            String errorCause = causeString(testable.errorCause());
+            return stage + exception + errorCause;
         }
         return "";
     }
diff --git a/src/cz/crcs/ectester/standalone/output/XMLTestWriter.java b/src/cz/crcs/ectester/standalone/output/XMLTestWriter.java
index 9332759..1458a12 100644
--- a/src/cz/crcs/ectester/standalone/output/XMLTestWriter.java
+++ b/src/cz/crcs/ectester/standalone/output/XMLTestWriter.java
@@ -104,10 +104,22 @@ public class XMLTestWriter extends BaseXMLTestWriter {
         return result;
     }
 
+    private String causeObject(Object cause) {
+        if (cause == null) {
+            return "";
+        } else if (cause instanceof Exception) {
+            Exception ex = ((Exception) cause);
+            return ex.getClass().getCanonicalName() + " : " + ex.getMessage();
+        } else {
+            return cause.toString();
+        }
+    }
+
     @Override
     protected Element testableElement(Testable t) {
         Element result = doc.createElement("test");
         if (t instanceof StandaloneTestable) {
+            StandaloneTestable<?> testable = (StandaloneTestable) t;
             if (t instanceof KeyGeneratorTestable) {
                 result.setAttribute("type", "key-pair-generator");
                 result.appendChild(kgtElement((KeyGeneratorTestable) t));
@@ -118,7 +130,10 @@ public class XMLTestWriter extends BaseXMLTestWriter {
                 result.setAttribute("type", "signature");
                 result.appendChild(sigElement((SignatureTestable) t));
             }
-            result.appendChild(stageElement((StandaloneTestable) t));
+            result.appendChild(stageElement(testable));
+            Element exception = doc.createElement("exception");
+            exception.setTextContent(causeObject(testable.getException()) + causeObject(testable.errorCause()));
+            result.appendChild(exception);
         }
         return result;
     }
diff --git a/src/cz/crcs/ectester/standalone/output/YAMLTestWriter.java b/src/cz/crcs/ectester/standalone/output/YAMLTestWriter.java
index 0926b98..433624a 100644
--- a/src/cz/crcs/ectester/standalone/output/YAMLTestWriter.java
+++ b/src/cz/crcs/ectester/standalone/output/YAMLTestWriter.java
@@ -16,6 +16,7 @@ import java.security.Key;
 import java.security.PrivateKey;
 import java.security.PublicKey;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 /**
@@ -27,7 +28,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     }
 
     private Map<String, Object> keyObject(Key key) {
-        Map<String, Object> kObject = new HashMap<>();
+        Map<String, Object> kObject = new LinkedHashMap<>();
         if (key == null) {
             return kObject;
         }
@@ -38,7 +39,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     }
 
     private Map<String, Object> kaObject(KeyAgreementTestable kat) {
-        Map<String, Object> katObject = new HashMap<>();
+        Map<String, Object> katObject = new LinkedHashMap<>();
         katObject.put("algo", kat.getKa().getAlgorithm());
         katObject.put("secret", ByteUtil.bytesToHex(kat.getSecret()));
 
@@ -51,10 +52,10 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     }
 
     private Map<String, Object> kgtObject(KeyGeneratorTestable kgt) {
-        Map<String, Object> kgtObject = new HashMap<>();
+        Map<String, Object> kgtObject = new LinkedHashMap<>();
         kgtObject.put("algo", kgt.getKpg().getAlgorithm());
 
-        Map<String, Object> keypair = new HashMap<>();
+        Map<String, Object> keypair = new LinkedHashMap<>();
         if (kgt.getKeyPair() != null) {
             PublicKey pkey = kgt.getKeyPair().getPublic();
             Map<String, Object> pubObject = keyObject(pkey);
@@ -70,17 +71,29 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     }
 
     private Map<String, Object> sigObject(SignatureTestable sig) {
-        Map<String, Object> sigObject = new HashMap<>();
+        Map<String, Object> sigObject = new LinkedHashMap<>();
         sigObject.put("algo", sig.getSig().getAlgorithm());
         sigObject.put("verified", sig.getVerified());
         sigObject.put("raw", ByteUtil.bytesToHex(sig.getSignature()));
         return sigObject;
     }
 
+    private String causeObject(Object cause) {
+        if (cause == null) {
+            return "";
+        } else if (cause instanceof Exception) {
+            Exception ex = ((Exception) cause);
+            return ex.getClass().getCanonicalName() + " : " + ex.getMessage();
+        } else {
+            return cause.toString();
+        }
+    }
+
     @Override
     protected Map<String, Object> testableObject(Testable t) {
-        Map<String, Object> result = new HashMap<>();
+        Map<String, Object> result = new LinkedHashMap<>();
         if (t instanceof StandaloneTestable) {
+            StandaloneTestable<?> testable = (StandaloneTestable) t;
             if (t instanceof KeyGeneratorTestable) {
                 result.put("type", "key-pair-generator");
                 result.put("key-pair-generator", kgtObject((KeyGeneratorTestable) t));
@@ -91,7 +104,8 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
                 result.put("type", "signature");
                 result.put("signature", sigObject((SignatureTestable) t));
             }
-            result.put("stage", ((StandaloneTestable)t).getStage().name());
+            result.put("stage", testable.getStage().name());
+            result.put("exception", causeObject(testable.getException()) + causeObject(testable.errorCause()));
         }
         return result;
     }
@@ -100,7 +114,7 @@ public class YAMLTestWriter extends BaseYAMLTestWriter {
     protected Map<String, Object> deviceObject(TestSuite suite) {
         if (suite instanceof StandaloneTestSuite) {
             StandaloneTestSuite standaloneSuite = (StandaloneTestSuite) suite;
-            Map<String, Object> result = new HashMap<>();
+            Map<String, Object> result = new LinkedHashMap<>();
             result.put("type", "library");
             result.put("ectester", ECTesterStandalone.VERSION);
             result.put("name", standaloneSuite.getLibrary().name());
diff --git a/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java b/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java
index ffcfc67..1447373 100644
--- a/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java
+++ b/src/cz/crcs/ectester/standalone/test/base/KeyAgreementTestable.java
@@ -87,8 +87,7 @@ public class KeyAgreementTestable extends StandaloneTestable<KeyAgreementTestabl
                     ka.init(privateKey);
                 }
             } catch (InvalidKeyException | InvalidAlgorithmParameterException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
@@ -96,17 +95,15 @@ public class KeyAgreementTestable extends StandaloneTestable<KeyAgreementTestabl
             try {
                 ka.doPhase(publicKey, true);
             } catch (IllegalStateException | InvalidKeyException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
             stage = KeyAgreementStage.GenerateSecret;
             try {
                 secret = ka.generateSecret();
-            } catch (IllegalStateException | UnsupportedOperationException isex) {
-                ok = false;
-                hasRun = true;
+            } catch (IllegalStateException | UnsupportedOperationException e) {
+                failOnException(e);
                 return;
             }
 
diff --git a/src/cz/crcs/ectester/standalone/test/base/KeyGeneratorTestable.java b/src/cz/crcs/ectester/standalone/test/base/KeyGeneratorTestable.java
index b561b8b..c05d6e3 100644
--- a/src/cz/crcs/ectester/standalone/test/base/KeyGeneratorTestable.java
+++ b/src/cz/crcs/ectester/standalone/test/base/KeyGeneratorTestable.java
@@ -47,8 +47,7 @@ public class KeyGeneratorTestable extends StandaloneTestable<KeyGeneratorTestabl
                     kpg.initialize(keysize);
                 }
             } catch (InvalidAlgorithmParameterException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
diff --git a/src/cz/crcs/ectester/standalone/test/base/SignatureTestable.java b/src/cz/crcs/ectester/standalone/test/base/SignatureTestable.java
index 873757b..b8db7b8 100644
--- a/src/cz/crcs/ectester/standalone/test/base/SignatureTestable.java
+++ b/src/cz/crcs/ectester/standalone/test/base/SignatureTestable.java
@@ -65,8 +65,7 @@ public class SignatureTestable extends StandaloneTestable<SignatureTestable.Sign
             try {
                 sig.initSign(signKey);
             } catch (InvalidKeyException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
@@ -74,8 +73,7 @@ public class SignatureTestable extends StandaloneTestable<SignatureTestable.Sign
             try {
                 sig.update(data);
             } catch (SignatureException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
@@ -83,8 +81,7 @@ public class SignatureTestable extends StandaloneTestable<SignatureTestable.Sign
             try {
                 signature = sig.sign();
             } catch (SignatureException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
@@ -92,8 +89,7 @@ public class SignatureTestable extends StandaloneTestable<SignatureTestable.Sign
             try {
                 sig.initVerify(verifyKey);
             } catch (InvalidKeyException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
@@ -101,8 +97,7 @@ public class SignatureTestable extends StandaloneTestable<SignatureTestable.Sign
             try {
                 sig.update(data);
             } catch (SignatureException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
                 return;
             }
 
@@ -110,11 +105,11 @@ public class SignatureTestable extends StandaloneTestable<SignatureTestable.Sign
             try {
                 verified = sig.verify(signature);
             } catch (SignatureException e) {
-                ok = false;
-                hasRun = true;
+                failOnException(e);
+                return;
             }
 
-            ok = true;
+            ok = verified;
         } catch (Exception ex) {
             ok = false;
             error = true;
diff --git a/src/cz/crcs/ectester/standalone/test/base/StandaloneTestable.java b/src/cz/crcs/ectester/standalone/test/base/StandaloneTestable.java
index 8654e94..47bffc1 100644
--- a/src/cz/crcs/ectester/standalone/test/base/StandaloneTestable.java
+++ b/src/cz/crcs/ectester/standalone/test/base/StandaloneTestable.java
@@ -7,8 +7,19 @@ import cz.crcs.ectester.common.test.BaseTestable;
  */
 public abstract class StandaloneTestable<T extends Enum<T>> extends BaseTestable {
     protected T stage;
+    protected Exception exception;
 
     public T getStage() {
         return stage;
     }
+
+    public Exception getException() {
+        return exception;
+    }
+
+    protected void failOnException(Exception ex) {
+        ok = false;
+        hasRun = true;
+        exception = ex;
+    }
 }