diff --git a/.github/workflows/gradle-ethereum-tests.yml b/.github/workflows/gradle-ethereum-tests.yml index c5d4c57d66..897178fc3f 100644 --- a/.github/workflows/gradle-ethereum-tests.yml +++ b/.github/workflows/gradle-ethereum-tests.yml @@ -52,10 +52,12 @@ jobs: - name: Upload test report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ethereum-tests-report - path: reference-tests/build/reports/tests/**/* + path: | + reference-tests/build/reports/tests/**/* + tmp/local/* ethereum-tests-go-corset: @@ -108,7 +110,9 @@ jobs: - name: Upload test report if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ethereum-tests-go-corset-report - path: reference-tests/build/reports/tests/**/* + path: | + reference-tests/build/reports/tests/**/* + tmp/local/* diff --git a/reference-tests/build.gradle b/reference-tests/build.gradle index 462f63f9ce..66a907500b 100644 --- a/reference-tests/build.gradle +++ b/reference-tests/build.gradle @@ -71,6 +71,8 @@ tasks.register('referenceGeneralStateTests', Test) { description = 'Runs ETH reference general state tests.' dependsOn generateGeneralStateReferenceTests + environment.put("REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "GeneralStateReferenceTestOutcome.json") + useJUnitPlatform { includeTags("GeneralStateReferenceTest") } diff --git a/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java index 958f0162b0..803378b171 100644 --- a/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java +++ b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java @@ -36,8 +36,14 @@ @Slf4j public class ReferenceTestOutcomeRecorderTool { - public static final String JSON_INPUT_FILENAME = "failedBlockchainReferenceTests-input.json"; - public static final String JSON_OUTPUT_FILENAME = "failedBlockchainReferenceTests.json"; + public static final String JSON_INPUT_FILENAME = + System.getenv() + .getOrDefault( + "REFERENCE_TEST_OUTCOME_INPUT_FILE", "failedBlockchainReferenceTests-input.json"); + public static final String JSON_OUTPUT_FILENAME = + System.getenv() + .getOrDefault( + "REFERENCE_TEST_OUTCOME_OUTPUT_FILE", "failedBlockchainReferenceTests.json"); public static JsonConverter jsonConverter = JsonConverter.builder().build(); private static volatile AtomicInteger failedCounter = new AtomicInteger(0); private static volatile AtomicInteger successCounter = new AtomicInteger(0); @@ -91,34 +97,37 @@ public static Map> extractConstraints(String message) { // case where corset sends constraint failed and the list of constraints if (message.contains("constraints failed:")) { - cleaned = - cleaned.substring( - message.indexOf("constraints failed:") + "constraints failed:".length()); - String[] constraints = cleaned.split(","); - for (int i = 0; i < constraints.length; i++) { - getPairFromString(constraints[i], pairs); + List lines = cleaned.lines().toList(); + for (String line : lines) { + if (line.contains("constraints failed:")) { + String[] failingConstraints = + line.substring(line.indexOf("constraints failed:") + "constraints failed:".length()) + .split(","); + for (String constraint : failingConstraints) { + getPairFromString(constraint, pairs); + } + } } } else if (message.contains("failing constraint")) { // case where corset sends failing constraint with constraints one by one - String[] lines = cleaned.split("\\n"); - for (int i = 0; i < lines.length; i++) { - if (lines[i].contains("failing constraint")) { - String line = - lines[i].substring( - lines[i].indexOf("failing constraint") + "failing constraint".length()); - line = line.replace(':', ' '); - getPairFromString(line, pairs); + List lines = cleaned.lines().toList(); + for (String line : lines) { + if (line.contains("failing constraint")) { + String failingConstraints = + line.substring(line.indexOf("failing constraint") + "failing constraint".length()) + .replace(':', ' '); + getPairFromString(failingConstraints, pairs); } } } else { // case where corset can't expend the trace if (message.contains("Error: while expanding ")) { - String[] lines = cleaned.split("\\n"); - for (int i = 0; i < lines.length; i++) { - if (lines[i].contains("reading data for")) { + List lines = cleaned.lines().toList(); + for (String line : lines) { + if (line.contains("reading data for")) { String regex = "for\\s+(\\w+)\\s+\\.\\s+(\\w+)"; Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(lines[i]); + Matcher matcher = pattern.matcher(line); if (matcher.find()) { String module = matcher.group(1); String constraint = matcher.group(2); @@ -198,7 +207,7 @@ private static CompletableFuture writeToJsonFileInternal(String name) { } return CompletableFuture.runAsync( () -> { - try (FileWriter file = new FileWriter(fileDirectory + name)) { + try (FileWriter file = new FileWriter(Path.of(fileDirectory, name).toString())) { objectMapper.writeValue( file, new BlockchainReferenceTestOutcome( diff --git a/reference-tests/src/test/java/net/consensys/linea/GeneralStateReferenceTestTools.java b/reference-tests/src/test/java/net/consensys/linea/GeneralStateReferenceTestTools.java index 6209588097..1eb422edd8 100644 --- a/reference-tests/src/test/java/net/consensys/linea/GeneralStateReferenceTestTools.java +++ b/reference-tests/src/test/java/net/consensys/linea/GeneralStateReferenceTestTools.java @@ -110,6 +110,10 @@ private static ProtocolSpec protocolSpec(final String name) { PARAMS.ignore("static_Call1MB1024Calldepth-\\w"); PARAMS.ignore("ShanghaiLove_.*"); PARAMS.ignore("VMTests/vmPerformance/"); + PARAMS.ignore("Call50000"); + PARAMS.ignore("static_LoopCallsDepthThenRevert3"); + PARAMS.ignore("Return50000"); + PARAMS.ignore("Callcode50000"); // Don't do time consuming tests PARAMS.ignore("CALLBlake2f_MaxRounds.*"); diff --git a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java index 37431b8795..031600b015 100644 --- a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java +++ b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java @@ -137,7 +137,16 @@ public void extractConstraints() { + "\n" + "Caused by:\n" + " constraints failed: \u001B[31m\u001B[1mrlptxrcpt.phase3\u001B[0m\u001B[39m, \u001B[31m\u001B[1mtxndata-into-wcp\u001B[0m\u001B[39m, \u001B[31m\u001B[1mtxndata-into-rlptxrcpt\u001B[0m\u001B[39m, \u001B[31m\u001B[1mtxndata.cumulative-gas\u001B[0m\u001B[39m, \u001B[31m\u001B[1mrlptxrcpt.phase-transition\u001B[0m\u001B[39m\n" - + "]"; + + "]" + + " at app//net.consensys.linea.testing.ExecutionEnvironment.checkTracer(ExecutionEnvironment.java:72)\n" + + " at app//net.consensys.linea.GeneralStateReferenceTestTools.executeTest(GeneralStateReferenceTestTools.java:235)\n" + + " at app//net.consensys.linea.generated.generalstate.GeneralStateReferenceTest_200.execution(GeneralStateReferenceTest_200.java:60)\n" + + " at java.base@21.0.2/java.lang.reflect.Method.invoke(Method.java:580)\n" + + " at java.base@21.0.2/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:387)\n" + + " at java.base@21.0.2/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1312)\n" + + " at java.base@21.0.2/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1843)\n" + + " at java.base@21.0.2/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1808)\n" + + " at java.base@21.0.2/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:188)"; Map> res = ReferenceTestOutcomeRecorderTool.extractConstraints(message); assertThat(res.size()).isEqualTo(2); diff --git a/reference-tests/src/test/resources/templates/GeneralStateReferenceTest.java.template b/reference-tests/src/test/resources/templates/GeneralStateReferenceTest.java.template index 81e136a3f6..03594a5d40 100644 --- a/reference-tests/src/test/resources/templates/GeneralStateReferenceTest.java.template +++ b/reference-tests/src/test/resources/templates/GeneralStateReferenceTest.java.template @@ -21,9 +21,11 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.util.stream.Stream; +import net.consensys.linea.ReferenceTestWatcher; import org.hyperledger.besu.ethereum.referencetests.GeneralStateTestCaseEipSpec; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.parallel.Execution; import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; @@ -31,11 +33,14 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; /** The general state test operation testing framework entry point. */ +@ExtendWith(ReferenceTestWatcher.class) @Tag("GeneralStateReferenceTest") @TestInstance(TestInstance.Lifecycle.PER_CLASS) @Execution(ExecutionMode.CONCURRENT) public class %%TESTS_NAME%% { + public ReferenceTestWatcher testWatcher = new ReferenceTestWatcher(); + private static final String[] TEST_CONFIG_FILE_DIR_PATH = new String[] {%%TESTS_FILE%%}; public static Stream getTestParametersForConfig() { diff --git a/testing/build.gradle b/testing/build.gradle index 7ff871af8f..5c714e3e95 100644 --- a/testing/build.gradle +++ b/testing/build.gradle @@ -40,8 +40,8 @@ dependencies { implementation 'org.assertj:assertj-core' } -nodeSetup { - mustRunAfter(spotlessBash, spotlessGroovy, spotlessSol) +node { + download = false } tasks.withType(GenerateContractWrappers.class).configureEach {