Make equivalence test less flaky.

2 files changed, 56 insertions, 30 deletions

diff --git a/test/gdb_script.py b/test/gdb_script.py
index c231a3c..93d4472 100644
--- a/test/gdb_script.py
+++ b/test/gdb_script.py
@@ -1,8 +1,10 @@
-import sys
+import os
 import json
 
 import gdb
 
+trace_file = open(os.environ["TRACE_FILE"], "w")
+
 
 def extract_bn(bn):
     data_ptr = bn["dp"]
@@ -32,7 +34,7 @@ class TraceFunction(gdb.Breakpoint):
             set_bp.enabled = True
             frame = gdb.newest_frame()
             block = frame.block()
-            print(frame.name(), file=sys.stderr)
+            print(frame.name(), flush=True, file=trace_file)
             out = []
             for sym in block:
                 if sym.is_argument:
@@ -47,11 +49,11 @@ class TraceFunction(gdb.Breakpoint):
                             out.append(deref)
                         else:
                             pt = extract_point(deref)
-                            print(f"{name}: {json.dumps(pt)}", file=sys.stderr)
+                            print(f"{name}: {json.dumps(pt)}", flush=True, file=trace_file)
             bp = TraceExit(frame)
             bp.silent = True
             bp.target = out
-        except RuntimeError as e:
+        except RuntimeError:
             pass
         return False  # Continue execution
 
@@ -60,7 +62,7 @@ class TraceExit(gdb.FinishBreakpoint):
     def stop(self):
         set_bp.enabled = False
         for i, point in enumerate(self.target):
-            print(f"out_{i}: {json.dumps(extract_point(point))}", file=sys.stderr)
+            print(f"out_{i}: {json.dumps(extract_point(point))}", flush=True, file=trace_file)
         return False  # Continue execution
 
 
@@ -84,4 +86,4 @@ set_bp = register_bp("point_set")
 set_bp.enabled = False
 
 gdb.execute("run")
-# print("\x04", file=sys.stderr)
+trace_file.close()
diff --git a/test/test_equivalence.py b/test/test_equivalence.py
index f808726..fcd5902 100644
--- a/test/test_equivalence.py
+++ b/test/test_equivalence.py
@@ -1,5 +1,6 @@
 import subprocess
 import json
+from tempfile import NamedTemporaryFile
 from typing import Generator, Any
 from click.testing import CliRunner
 
@@ -21,17 +22,37 @@ from pyecsca.ec.mult import WindowBoothMultiplier
 class GDBTarget(ImplTarget, BinaryTarget):
     def __init__(self, model: CurveModel, coords: CoordinateModel, **kwargs):
         super().__init__(model, coords, **kwargs)
+        self.trace_file = None
 
     def connect(self):
+        self.trace_file = NamedTemporaryFile("r+")
         with resources.path("test", "gdb_script.py") as gdb_script:
             self.process = subprocess.Popen(
-                ["gdb", "-batch", "-x", gdb_script, "--args", *self.binary],
+                [
+                    "gdb",
+                    "-q",
+                    "-batch-silent",
+                    "-x",
+                    gdb_script,
+                    "--args",
+                    *self.binary,
+                ],
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL,
+                env={
+                    "TRACE_FILE": self.trace_file.name,
+                },
                 text=True,
                 bufsize=1,
             )
 
+    def disconnect(self):
+        super().disconnect()
+        if self.trace_file is not None:
+            self.trace_file.close()
+            self.trace_file = None
+
 
 @pytest.fixture(scope="module")
 def target(simple_multiplier, secp128r1) -> Generator[GDBTarget, Any, None]:
@@ -132,32 +153,35 @@ def make_hashable(trace):
     return tuple(result)
 
 
-def test_equivalence(target, secp128r1, capfd):
+def test_equivalence(target, secp128r1):
     mult = target.mult
-    target.connect()
-    target.set_params(secp128r1)
-    for _ in range(3):
+    for i in range(10):
+        target.connect()
+        target.init_prng(i.to_bytes(4, "big"))
+        target.set_params(secp128r1)
+
         priv, pub = target.generate()
+
         with local(DefaultContext()) as ctx:
             mult.init(secp128r1, secp128r1.generator)
             expected = mult.multiply(priv).to_affine()
-        captured = capfd.readouterr()
-        with capfd.disabled():
-            assert secp128r1.curve.is_on_curve(pub)
-            assert pub == expected
-            from_codegen = parse_trace(captured.err)
-            from_sim = parse_ctx(ctx.actions[0]) + parse_ctx(ctx.actions[1])
-            codegen_set = set(make_hashable(from_codegen))
-            sim_set = set(make_hashable(from_sim))
-            if codegen_set != sim_set:
-                print(len(from_codegen), len(from_sim))
-                print("In codegen but not in sim:")
-                for entry in codegen_set - sim_set:
-                    print(entry)
-                print("In sim but not in codegen:")
-                for entry in sim_set - codegen_set:
-                    print(entry)
-            assert from_codegen == from_sim
 
-    target.quit()
-    target.disconnect()
+        assert secp128r1.curve.is_on_curve(pub)
+        assert pub == expected
+        err = target.trace_file.read()
+        from_codegen = parse_trace(err)
+        from_sim = parse_ctx(ctx.actions[0]) + parse_ctx(ctx.actions[1])
+        codegen_set = set(make_hashable(from_codegen))
+        sim_set = set(make_hashable(from_sim))
+        if codegen_set != sim_set:
+            print(len(from_codegen), len(from_sim))
+            print("In codegen but not in sim:")
+            for entry in codegen_set - sim_set:
+                print(entry)
+            print("In sim but not in codegen:")
+            for entry in sim_set - codegen_set:
+                print(entry)
+        assert from_codegen == from_sim
+
+        target.quit()
+        target.disconnect()