Merge pull request #33 from crocs-muni/fix/card-runtest

Fix test selection

3 files changed, 313 insertions, 0 deletions

diff --git a/reader/src/test/java/cz/crcs/ectester/reader/AppTests.java b/reader/src/test/java/cz/crcs/ectester/reader/AppTests.java
new file mode 100644
index 0000000..173f7e2
--- /dev/null
+++ b/reader/src/test/java/cz/crcs/ectester/reader/AppTests.java
@@ -0,0 +1,141 @@
+package cz.crcs.ectester.reader;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junitpioneer.jupiter.StdIo;
+import org.junitpioneer.jupiter.StdOut;
+
+import java.time.Duration;
+
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class AppTests {
+
+    @Test
+    @StdIo()
+    public void help(StdOut out) {
+        ECTesterReader.main(new String[]{"-h"});
+        String s = out.capturedString();
+        assertTrue(s.contains("ECTesterReader"));
+    }
+
+    @Test
+    @StdIo()
+    public void listSuites(StdOut out) {
+        ECTesterReader.main(new String[]{"--list-suites"});
+        String s = out.capturedString();
+        assertTrue(s.contains("default test suite"));
+    }
+
+    @Test
+    @StdIo()
+    public void listData(StdOut out) {
+        ECTesterReader.main(new String[]{"--list-named"});
+        String s = out.capturedString();
+        assertTrue(s.contains("secg"));
+    }
+
+    // Add StdIo to all the suite tests when this is resolved: https://github.com/junit-pioneer/junit-pioneer/issues/822
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void defaultSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "default", "-s"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    @Disabled
+    public void testVectorSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "test-vectors", "-s"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void compressionSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "compression", "-s"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    @Disabled
+    public void wrongSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "wrong", "-s", "-y"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void degenerateSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "degenerate", "-s", "-y"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    @Disabled
+    public void cofactorSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "cofactor", "-s", "-y"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    @Disabled
+    public void compositeSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "composite", "-s", "-y"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void invalidSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "invalid", "-s", "-y"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void edgeCasesSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "edge-cases", "-s", "-y"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void signatureSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "signature", "-s"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void twistSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "twist", "-s", "-y"}));
+    }
+
+    @Test
+    @XFail(value = "JCardSim sometimes times-out.")
+    public void miscellaneousSuite() {
+        assertTimeoutPreemptively(Duration.ofSeconds(60), () -> ECTesterReader.main(new String[]{"-t", "miscellaneous", "-s", "-y"}));
+    }
+
+    @Test
+    public void generate() {
+        ECTesterReader.main(new String[]{"-g", "10", "-s", "-fp", "-o", "/dev/null", "-b", "256"});
+    }
+
+    @Test
+    public void ecdh() {
+        ECTesterReader.main(new String[]{"-dh", "-fp", "-b", "256", "-s"});
+    }
+
+    @Test
+    public void ecdsa() {
+        ECTesterReader.main(new String[]{"-dsa", "-fp", "-b", "256", "-s"});
+    }
+
+    @Test
+    public void export() {
+        ECTesterReader.main(new String[]{"-e", "-fp", "-b", "256", "-s", "-o", "/dev/null"});
+    }
+
+    @Test
+    public void info() {
+        ECTesterReader.main(new String[]{"-nf", "-s"});
+    }
+}
diff --git a/reader/src/test/java/cz/crcs/ectester/reader/XFail.java b/reader/src/test/java/cz/crcs/ectester/reader/XFail.java
new file mode 100644
index 0000000..f42f530
--- /dev/null
+++ b/reader/src/test/java/cz/crcs/ectester/reader/XFail.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2016-2023 the original author or authors.
+ * Taken from https://github.com/junit-pioneer/junit-pioneer/blob/98cef28462c8b7ab66231cc5b7e8daef3b329f67/src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package cz.crcs.ectester.reader;
+
+import static java.lang.annotation.ElementType.ANNOTATION_TYPE;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * {@code @ExpectedToFail} is a JUnit Jupiter extension to mark test methods as temporarily
+ * 'expected to fail'. Such test methods will still be executed but when they result in a test
+ * failure or error the test will be aborted. However, if the test method unexpectedly executes
+ * successfully, it is marked as failure to let the developer know that the test is now
+ * successful and that the {@code @ExpectedToFail} annotation can be removed.
+ *
+ * <p>The big difference compared to JUnit's {@link org.junit.jupiter.api.Disabled @Disabled}
+ * annotation is that the developer is informed as soon as a test is successful again.
+ * This helps to avoid creating duplicate tests by accident and counteracts the accumulation
+ * of disabled tests over time.</p>
+ *
+ * <p>Further, the {@link #withExceptions()} attribute can be used to restrict the extension's behavior
+ * to specific exceptions. That is, only if the test method ends up throwing one of the specified exceptions
+ * will the test be aborted. This can, for example, be used when the production code temporarily throws
+ * an {@link UnsupportedOperationException} because some feature has not been implemented yet, but the
+ * test method is already implemented and should not fail on a failing assertion.
+ * </p>
+ *
+ * <p>The annotation can only be used on methods and as meta-annotation on other annotation types.
+ * Similar to {@code @Disabled}, it has to be used in addition to a "testable" annotation, such
+ * as {@link org.junit.jupiter.api.Test @Test}. Otherwise the annotation has no effect.</p>
+ *
+ * <p><b>Important:</b> This annotation is <b>not</b> intended as a way to mark test methods
+ * which intentionally cause exceptions. Such test methods should use
+ * {@link org.junit.jupiter.api.Assertions#assertThrows(Class, org.junit.jupiter.api.function.Executable) assertThrows}
+ * or similar means to explicitly test for a specific exception class being thrown by a
+ * specific action.</p>
+ *
+ * <p>For more details and examples, see
+ * <a href="https://junit-pioneer.org/docs/expected-to-fail-tests/" target="_top">the documentation on <code>@ExpectedToFail</code></a>.</p>
+ *
+ * @since 1.8.0
+ * @see org.junit.jupiter.api.Disabled
+ */
+@Documented
+@Retention(RUNTIME)
+/*
+ * Only supports METHOD and ANNOTATION_TYPE as targets but not test classes because there
+ * it is not clear what the 'correct' behavior would be when only a few test methods
+ * execute successfully. Would the developer then have to remove the @ExpectedToFail annotation
+ * from the test class and annotate methods individually?
+ */
+@Target({ METHOD, ANNOTATION_TYPE })
+@ExtendWith(XFailExtension.class)
+public @interface XFail {
+
+    /**
+     * Defines the message to show when a test is aborted because it is failing.
+     * This can be used for example to briefly explain why the tested code is not working
+     * as intended at the moment.
+     * An empty string (the default) causes a generic default message to be used.
+     */
+    String value() default "";
+
+    /**
+     * Specifies which exceptions are expected to be thrown and will cause the test to be aborted rather than fail.
+     * An empty array is considered a configuration error and will cause the test to fail. Instead, consider leaving
+     * the attribute set to the default value when any exception should cause the test to be aborted.
+     */
+    Class<? extends Throwable>[] withExceptions() default { Throwable.class };
+
+}
\ No newline at end of file
diff --git a/reader/src/test/java/cz/crcs/ectester/reader/XFaillExtension.java b/reader/src/test/java/cz/crcs/ectester/reader/XFaillExtension.java
new file mode 100644
index 0000000..b190e1a
--- /dev/null
+++ b/reader/src/test/java/cz/crcs/ectester/reader/XFaillExtension.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2016-2023 the original author or authors.
+ * Taken from https://github.com/junit-pioneer/junit-pioneer/blob/98cef28462c8b7ab66231cc5b7e8daef3b329f67/src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package cz.crcs.ectester.reader;
+
+import java.lang.reflect.Method;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionConfigurationException;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.InvocationInterceptor;
+import org.junit.jupiter.api.extension.ReflectiveInvocationContext;
+import org.junit.platform.commons.support.AnnotationSupport;
+import org.opentest4j.AssertionFailedError;
+import org.opentest4j.TestAbortedException;
+
+class XFailExtension implements Extension, InvocationInterceptor {
+
+    @Override
+    public void interceptTestMethod(Invocation<Void> invocation, ReflectiveInvocationContext<Method> invocationContext,
+                                    ExtensionContext extensionContext) throws Throwable {
+        invokeAndInvertResult(invocation, extensionContext);
+    }
+
+    private static void invokeAndInvertResult(Invocation<Void> invocation, ExtensionContext extensionContext)
+            throws Throwable {
+        XFail expectedToFail = getExpectedToFailAnnotation(extensionContext);
+        if (expectedToFail.withExceptions().length == 0) {
+            throw new ExtensionConfigurationException("@XFail withExceptions must not be empty");
+        }
+
+        try {
+            invocation.proceed();
+            // at this point, the invocation succeeded, so we'd want to call `fail(...)`,
+            // but that would get handled by the following `catch` and so it's easier
+            // to instead fall through to a `fail(...)` after the `catch` block
+        }
+        catch (Throwable t) {
+            if (shouldPreserveException(t)) {
+                throw t;
+            }
+
+            if (Stream.of(expectedToFail.withExceptions()).noneMatch(clazz -> clazz.isInstance(t))) {
+                throw new AssertionFailedError(
+                        "Test marked as temporarily 'expected to fail' failed with an unexpected exception", t);
+            }
+
+            String message = expectedToFail.value();
+            if (message.isEmpty()) {
+                message = "Test marked as temporarily 'expected to fail' failed as expected";
+            }
+
+            throw new TestAbortedException(message, t);
+        }
+    }
+
+    /**
+     * Returns whether the exception should be preserved and reported as is instead
+     * of considering it an 'expected to fail' exception.
+     *
+     * <p>This method is used for exceptions that abort test execution and should
+     * have higher precedence than aborted exceptions thrown by this extension.</p>
+     */
+    private static boolean shouldPreserveException(Throwable t) {
+        // Note: Ideally would use the same logic JUnit uses to determine if exception is aborting
+        // execution, see its class OpenTest4JAndJUnit4AwareThrowableCollector
+        return t instanceof TestAbortedException;
+    }
+
+    private static XFail getExpectedToFailAnnotation(ExtensionContext context) {
+        return AnnotationSupport
+                .findAnnotation(context.getRequiredTestMethod(), XFail.class)
+                .orElseThrow(() -> new IllegalStateException("@XFail is missing."));
+
+    }
+
+}
\ No newline at end of file