From d962e8ea1d56bc08ba729e5922ea559bf4e4f5d2 Mon Sep 17 00:00:00 2001
From: "Jurgen J. Vinju" <Jurgen.Vinju@cwi.nl>
Date: Mon, 15 Jan 2024 17:08:55 +0100
Subject: [PATCH] tests that fail with unexpected exceptions, and have random
 parameters, are now properly reported on

---
 .../interpreter/DefaultTestResultListener.java    | 14 +-------------
 src/org/rascalmpl/interpreter/TestEvaluator.java  |  6 +++++-
 .../rascalmpl/test/infrastructure/QuickCheck.java | 15 +++++++++++----
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/src/org/rascalmpl/interpreter/DefaultTestResultListener.java b/src/org/rascalmpl/interpreter/DefaultTestResultListener.java
index d588ab41cb1..8cc036b3dd1 100644
--- a/src/org/rascalmpl/interpreter/DefaultTestResultListener.java
+++ b/src/org/rascalmpl/interpreter/DefaultTestResultListener.java
@@ -119,19 +119,7 @@ else if (t != null) {
 		        err.println();
 		    }
 		    err.println("error: " + test + " @ " + ReplTextWriter.valueToString(loc));
-		    err.println("\t" + t.getMessage());
-		    
-		    if (t instanceof Throw) {
-		        try {
-                    ((Throw) t).getTrace().prettyPrintedString(err, new StandardTextWriter(true));
-                }
-                catch (IOException e) {
-                    // should not happen
-                }
-		    }
-		    else {
-		        t.printStackTrace(err);
-		    }
+		    err.println(message);
 		}
 		else {
 		    failures++;
diff --git a/src/org/rascalmpl/interpreter/TestEvaluator.java b/src/org/rascalmpl/interpreter/TestEvaluator.java
index 47da78a248c..c555402e864 100644
--- a/src/org/rascalmpl/interpreter/TestEvaluator.java
+++ b/src/org/rascalmpl/interpreter/TestEvaluator.java
@@ -20,12 +20,14 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 
 import org.rascalmpl.interpreter.env.ModuleEnvironment;
 import org.rascalmpl.interpreter.result.AbstractFunction;
 import org.rascalmpl.test.infrastructure.QuickCheck;
 import org.rascalmpl.test.infrastructure.QuickCheck.TestResult;
+import org.rascalmpl.test.infrastructure.QuickCheck.UnExpectedExceptionThrownResult;
 
 import io.usethesource.vallang.IBool;
 import io.usethesource.vallang.IString;
@@ -117,7 +119,8 @@ private void runTests(ModuleEnvironment env, List<AbstractFunction> tests) {
                         }
                     }
                     catch (Throwable e) {
-                        return new TestResult(false, e);
+                        // TODO: add bound type parameters
+                        return new UnExpectedExceptionThrownResult(test.getEnv().getName() + "::" + test.getName(), actuals, Map.of(), args, e);
                     }
                 }, env.getRoot().getStore(), tries, maxDepth, maxWidth);
                 
@@ -137,6 +140,7 @@ private void runTests(ModuleEnvironment env, List<AbstractFunction> tests) {
             catch(Throwable e){
                 testResultListener.report(false, test.getName(), test.getAst().getLocation(), e.getMessage(), e);
             }
+            
             eval.getOutPrinter().flush();
             eval.getErrorPrinter().flush();
         }
diff --git a/src/org/rascalmpl/test/infrastructure/QuickCheck.java b/src/org/rascalmpl/test/infrastructure/QuickCheck.java
index 25ba7105bd1..0b9fbd1167b 100644
--- a/src/org/rascalmpl/test/infrastructure/QuickCheck.java
+++ b/src/org/rascalmpl/test/infrastructure/QuickCheck.java
@@ -12,6 +12,7 @@
  */ 
 package org.rascalmpl.test.infrastructure;
 
+import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.HashMap;
 import java.util.Map;
@@ -25,6 +26,7 @@
 import io.usethesource.vallang.IString;
 import io.usethesource.vallang.IValue;
 import io.usethesource.vallang.IValueFactory;
+import io.usethesource.vallang.io.StandardTextWriter;
 import io.usethesource.vallang.type.Type;
 import io.usethesource.vallang.type.TypeStore;
 
@@ -196,7 +198,7 @@ public void writeMessage(PrintWriter out) {
         }
     }
 
-    public class UnExpectedExceptionThrownResult extends TestFailedResult {
+    public static class UnExpectedExceptionThrownResult extends TestFailedResult {
 
         public UnExpectedExceptionThrownResult(String functionName, Type[] actualTypes,
             Map<Type, Type> tpbindings, IValue[] values, Throwable thrownException) {
@@ -210,16 +212,21 @@ public void writeMessage(PrintWriter out) {
             out.println("Exception:");
             if (thrownException instanceof Throw) {
                 out.println(((Throw)thrownException).getMessage());
+                try {
+                    ((Throw) thrownException).getTrace().prettyPrintedString(out, new StandardTextWriter(true));
+                }
+                catch (IOException e) {
+                    // should not happen
+                }
             }
             else {
-                out.println(thrownException.toString());
+                // out.println(thrownException.toString());
                 thrownException.printStackTrace(out);
             }
         }
-
     }
 
-    public class ExceptionNotThrownResult extends TestFailedResult {
+    public static class ExceptionNotThrownResult extends TestFailedResult {
         public ExceptionNotThrownResult(String functionName, Type[] actualTypes,
             Map<Type, Type> tpbindings, IValue[] values, String expected) {
             super(functionName, "test did not throw '" + expected + "' exception", actualTypes, tpbindings, values);