Skip GeneralStateReferencetest with high memory usage and enable test…

… outcome recorder for generalStateReferenceTests (#1438) * Skip GeneralStateReferencetest with high memory usage and enable test outcome recorder for generalStateReferenceTests * Fix test tool * fix spotless * revert timeout changes * Cleanup * Do not download node as it is not used
Consensys · Oct 22, 2024 · 2978c7f · 2978c7f
1 parent 4205f90
commit 2978c7f
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 28 deletions.
diff --git a/.github/workflows/gradle-ethereum-tests.yml b/.github/workflows/gradle-ethereum-tests.yml
@@ -52,10 +52,12 @@ jobs:
 
       - name: Upload test report
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ethereum-tests-report
-          path: reference-tests/build/reports/tests/**/*
+          path: |
+            reference-tests/build/reports/tests/**/*
+            tmp/local/*
 
 
   ethereum-tests-go-corset:
@@ -108,7 +110,9 @@ jobs:
 
       - name: Upload test report
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ethereum-tests-go-corset-report
-          path: reference-tests/build/reports/tests/**/*
+          path: | 
+            reference-tests/build/reports/tests/**/*
+            tmp/local/*
diff --git a/reference-tests/build.gradle b/reference-tests/build.gradle
@@ -71,6 +71,8 @@ tasks.register('referenceGeneralStateTests', Test) {
   description = 'Runs ETH reference general state tests.'
   dependsOn generateGeneralStateReferenceTests
 
+  environment.put("REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "GeneralStateReferenceTestOutcome.json")
+
   useJUnitPlatform {
     includeTags("GeneralStateReferenceTest")
   }

diff --git a/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java
@@ -36,8 +36,14 @@
 @Slf4j
 public class ReferenceTestOutcomeRecorderTool {
 
-  public static final String JSON_INPUT_FILENAME = "failedBlockchainReferenceTests-input.json";
-  public static final String JSON_OUTPUT_FILENAME = "failedBlockchainReferenceTests.json";
+  public static final String JSON_INPUT_FILENAME =
+      System.getenv()
+          .getOrDefault(
+              "REFERENCE_TEST_OUTCOME_INPUT_FILE", "failedBlockchainReferenceTests-input.json");
+  public static final String JSON_OUTPUT_FILENAME =
+      System.getenv()
+          .getOrDefault(
+              "REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "failedBlockchainReferenceTests.json");
   public static JsonConverter jsonConverter = JsonConverter.builder().build();
   private static volatile AtomicInteger failedCounter = new AtomicInteger(0);
   private static volatile AtomicInteger successCounter = new AtomicInteger(0);
@@ -91,34 +97,37 @@ public static Map<String, Set<String>> extractConstraints(String message) {
 
     // case where corset sends constraint failed and the list of constraints
     if (message.contains("constraints failed:")) {
-      cleaned =
-          cleaned.substring(
-              message.indexOf("constraints failed:") + "constraints failed:".length());
-      String[] constraints = cleaned.split(",");
-      for (int i = 0; i < constraints.length; i++) {
-        getPairFromString(constraints[i], pairs);
+      List<String> lines = cleaned.lines().toList();
+      for (String line : lines) {
+        if (line.contains("constraints failed:")) {
+          String[] failingConstraints =
+              line.substring(line.indexOf("constraints failed:") + "constraints failed:".length())
+                  .split(",");
+          for (String constraint : failingConstraints) {
+            getPairFromString(constraint, pairs);
+          }
+        }
       }
     } else if (message.contains("failing constraint")) {
       // case where corset sends failing constraint with constraints one by one
-      String[] lines = cleaned.split("\\n");
-      for (int i = 0; i < lines.length; i++) {
-        if (lines[i].contains("failing constraint")) {
-          String line =
-              lines[i].substring(
-                  lines[i].indexOf("failing constraint") + "failing constraint".length());
-          line = line.replace(':', ' ');
-          getPairFromString(line, pairs);
+      List<String> lines = cleaned.lines().toList();
+      for (String line : lines) {
+        if (line.contains("failing constraint")) {
+          String failingConstraints =
+              line.substring(line.indexOf("failing constraint") + "failing constraint".length())
+                  .replace(':', ' ');
+          getPairFromString(failingConstraints, pairs);
         }
       }
     } else {
       // case where corset can't expend the trace
       if (message.contains("Error: while expanding ")) {
-        String[] lines = cleaned.split("\\n");
-        for (int i = 0; i < lines.length; i++) {
-          if (lines[i].contains("reading data for")) {
+        List<String> lines = cleaned.lines().toList();
+        for (String line : lines) {
+          if (line.contains("reading data for")) {
             String regex = "for\\s+(\\w+)\\s+\\.\\s+(\\w+)";
             Pattern pattern = Pattern.compile(regex);
-            Matcher matcher = pattern.matcher(lines[i]);
+            Matcher matcher = pattern.matcher(line);
             if (matcher.find()) {
               String module = matcher.group(1);
               String constraint = matcher.group(2);
@@ -198,7 +207,7 @@ private static CompletableFuture<Void> writeToJsonFileInternal(String name) {
     }
     return CompletableFuture.runAsync(
         () -> {
-          try (FileWriter file = new FileWriter(fileDirectory + name)) {
+          try (FileWriter file = new FileWriter(Path.of(fileDirectory, name).toString())) {
             objectMapper.writeValue(
                 file,
                 new BlockchainReferenceTestOutcome(

diff --git a/reference-tests/src/test/java/net/consensys/linea/GeneralStateReferenceTestTools.java b/reference-tests/src/test/java/net/consensys/linea/GeneralStateReferenceTestTools.java
@@ -110,6 +110,10 @@ private static ProtocolSpec protocolSpec(final String name) {
     PARAMS.ignore("static_Call1MB1024Calldepth-\\w");
     PARAMS.ignore("ShanghaiLove_.*");
     PARAMS.ignore("VMTests/vmPerformance/");
+    PARAMS.ignore("Call50000");
+    PARAMS.ignore("static_LoopCallsDepthThenRevert3");
+    PARAMS.ignore("Return50000");
+    PARAMS.ignore("Callcode50000");
 
     // Don't do time consuming tests
     PARAMS.ignore("CALLBlake2f_MaxRounds.*");

diff --git a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java
@@ -137,7 +137,16 @@ public void extractConstraints() {
             + "\n"
             + "Caused by:\n"
             + "    constraints failed: \u001B[31m\u001B[1mrlptxrcpt.phase3\u001B[0m\u001B[39m, \u001B[31m\u001B[1mtxndata-into-wcp\u001B[0m\u001B[39m, \u001B[31m\u001B[1mtxndata-into-rlptxrcpt\u001B[0m\u001B[39m, \u001B[31m\u001B[1mtxndata.cumulative-gas\u001B[0m\u001B[39m, \u001B[31m\u001B[1mrlptxrcpt.phase-transition\u001B[0m\u001B[39m\n"
-            + "]";
+            + "]"
+            + "      at app//net.consensys.linea.testing.ExecutionEnvironment.checkTracer(ExecutionEnvironment.java:72)\n"
+            + "      at app//net.consensys.linea.GeneralStateReferenceTestTools.executeTest(GeneralStateReferenceTestTools.java:235)\n"
+            + "      at app//net.consensys.linea.generated.generalstate.GeneralStateReferenceTest_200.execution(GeneralStateReferenceTest_200.java:60)\n"
+            + "      at [email protected]/java.lang.reflect.Method.invoke(Method.java:580)\n"
+            + "      at [email protected]/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:387)\n"
+            + "      at [email protected]/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1312)\n"
+            + "      at [email protected]/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1843)\n"
+            + "      at [email protected]/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1808)\n"
+            + "      at [email protected]/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:188)";
 
     Map<String, Set<String>> res = ReferenceTestOutcomeRecorderTool.extractConstraints(message);
     assertThat(res.size()).isEqualTo(2);

diff --git a/reference-tests/src/test/resources/templates/GeneralStateReferenceTest.java.template b/reference-tests/src/test/resources/templates/GeneralStateReferenceTest.java.template
@@ -21,21 +21,26 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import java.util.stream.Stream;
 
+import net.consensys.linea.ReferenceTestWatcher;
 import org.hyperledger.besu.ethereum.referencetests.GeneralStateTestCaseEipSpec;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
 /** The general state test operation testing framework entry point. */
+@ExtendWith(ReferenceTestWatcher.class)
 @Tag("GeneralStateReferenceTest")
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 @Execution(ExecutionMode.CONCURRENT)
 public class %%TESTS_NAME%% {
 
+  public ReferenceTestWatcher testWatcher = new ReferenceTestWatcher();
+
   private static final String[] TEST_CONFIG_FILE_DIR_PATH = new String[] {%%TESTS_FILE%%};
 
   public static Stream<Arguments> getTestParametersForConfig() {

diff --git a/testing/build.gradle b/testing/build.gradle
@@ -40,8 +40,8 @@ dependencies {
   implementation 'org.assertj:assertj-core'
 }
 
-nodeSetup {
-  mustRunAfter(spotlessBash, spotlessGroovy, spotlessSol)
+node {
+  download = false
 }
 
 tasks.withType(GenerateContractWrappers.class).configureEach {