diff --git a/reference-tests/src/main/java/net/consensys/linea/FailedTestJson.java b/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java similarity index 95% rename from reference-tests/src/main/java/net/consensys/linea/FailedTestJson.java rename to reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java index d38f7b0631..19e796de36 100644 --- a/reference-tests/src/main/java/net/consensys/linea/FailedTestJson.java +++ b/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestJson.java @@ -26,11 +26,11 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class FailedTestJson { +public class BlockchainReferenceTestJson { static String fileDirectory = setFileDirectory(); @Synchronized - public static CompletableFuture readFailedTestsOutput(String fileName) { + public static CompletableFuture readBlockchainReferenceTestsOutput(String fileName) { return CompletableFuture.supplyAsync( () -> { Path directoryPath = Paths.get(fileDirectory); diff --git a/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java b/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java new file mode 100644 index 0000000000..5bd84b1ddb --- /dev/null +++ b/reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java @@ -0,0 +1,23 @@ +/* + * Copyright Consensys Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package net.consensys.linea; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({"failedCounter", "successCounter", "modules"}) +public record BlockchainReferenceTestOutcome( + int failedCounter, int successCounter, List modulesToConstraints) {} diff --git a/reference-tests/src/main/java/net/consensys/linea/ModuleToConstraints.java b/reference-tests/src/main/java/net/consensys/linea/ModuleToConstraints.java index 3fa2e20518..89f6a1fb86 100644 --- a/reference-tests/src/main/java/net/consensys/linea/ModuleToConstraints.java +++ b/reference-tests/src/main/java/net/consensys/linea/ModuleToConstraints.java @@ -21,17 +21,9 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; public record ModuleToConstraints(String moduleName, Map> constraints) { - public ModuleToConstraints( - @JsonProperty("moduleName") String moduleName, - @JsonProperty("constraints") Map> constraints) { - this.moduleName = moduleName; - this.constraints = constraints; - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/reference-tests/src/main/java/net/consensys/linea/MapFailedReferenceTestsTool.java b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java similarity index 69% rename from reference-tests/src/main/java/net/consensys/linea/MapFailedReferenceTestsTool.java rename to reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java index 1fbdd36332..b52d5ca6a4 100644 --- a/reference-tests/src/main/java/net/consensys/linea/MapFailedReferenceTestsTool.java +++ b/reference-tests/src/main/java/net/consensys/linea/ReferenceTestOutcomeRecorderTool.java @@ -14,8 +14,8 @@ */ package net.consensys.linea; -import static net.consensys.linea.FailedTestJson.readFailedTestsOutput; -import static net.consensys.linea.FailedTestJson.writeToJsonFile; +import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput; +import static net.consensys.linea.BlockchainReferenceTestJson.writeToJsonFile; import java.util.ArrayList; import java.util.Arrays; @@ -33,31 +33,58 @@ import net.consensys.linea.zktracer.json.JsonConverter; @Slf4j -public class MapFailedReferenceTestsTool { +public class ReferenceTestOutcomeRecorderTool { + + static JsonConverter jsonConverter = JsonConverter.builder().build(); @Synchronized public static void mapAndStoreFailedReferenceTest( String testName, List logEventMessages, String jsonOutputFilename) { Set failedConstraints = getFailedConstraints(logEventMessages); if (!failedConstraints.isEmpty()) { - readFailedTestsOutput(jsonOutputFilename) + readBlockchainReferenceTestsOutput(jsonOutputFilename) .thenCompose( jsonString -> { JsonConverter jsonConverter = JsonConverter.builder().build(); - List modulesToConstraints = - getModulesToConstraints(jsonString, jsonConverter); + BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = + getBlockchainReferenceTestOutcome(jsonString); mapFailedConstraintsToTestsToModule( - modulesToConstraints, failedConstraints, testName); - - jsonString = jsonConverter.toJson(modulesToConstraints); + blockchainReferenceTestOutcome.modulesToConstraints(), + failedConstraints, + testName); + + BlockchainReferenceTestOutcome finalBlockchainReferenceTestOutcome = + new BlockchainReferenceTestOutcome( + blockchainReferenceTestOutcome.failedCounter() + 1, + blockchainReferenceTestOutcome.successCounter(), + blockchainReferenceTestOutcome.modulesToConstraints()); + jsonString = jsonConverter.toJson(finalBlockchainReferenceTestOutcome); writeToJsonFile(jsonString, jsonOutputFilename); return null; }); } } + @Synchronized + public static void incrementSuccessRate(String jsonOutputFilename) { + readBlockchainReferenceTestsOutput(jsonOutputFilename) + .thenAccept( + jsonString -> { + BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = + getBlockchainReferenceTestOutcome(jsonString); + + BlockchainReferenceTestOutcome finalBlockchainReferenceTestOutcome = + new BlockchainReferenceTestOutcome( + blockchainReferenceTestOutcome.failedCounter(), + blockchainReferenceTestOutcome.successCounter() + 1, + blockchainReferenceTestOutcome.modulesToConstraints()); + jsonString = jsonConverter.toJson(finalBlockchainReferenceTestOutcome); + writeToJsonFile(jsonString, jsonOutputFilename); + }); + } + @Synchronized private static void mapFailedConstraintsToTestsToModule( List modulesToConstraints, @@ -119,15 +146,16 @@ private static void addModuleIfAbsent( } @Synchronized - public static List getModulesToConstraints( - String jsonString, JsonConverter jsonConverter) { - List moduleToConstraints = new ArrayList<>(); + public static BlockchainReferenceTestOutcome getBlockchainReferenceTestOutcome( + String jsonString) { + BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = null; if (!jsonString.isEmpty()) { - moduleToConstraints = - new ArrayList<>( - Arrays.asList(jsonConverter.fromJson(jsonString, ModuleToConstraints[].class))); + blockchainReferenceTestOutcome = + jsonConverter.fromJson(jsonString, BlockchainReferenceTestOutcome.class); + } else { + blockchainReferenceTestOutcome = new BlockchainReferenceTestOutcome(0, 0, new ArrayList<>()); } - return moduleToConstraints; + return blockchainReferenceTestOutcome; } private static Set getFailedConstraints(List logEventMessages) { diff --git a/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java b/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java index 168ce285f0..de46dbf7c6 100644 --- a/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java +++ b/reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java @@ -15,9 +15,8 @@ package net.consensys.linea; -import static net.consensys.linea.FailedTestJson.readFailedTestsOutput; -import static net.consensys.linea.MapFailedReferenceTestsTool.getModule; -import static net.consensys.linea.MapFailedReferenceTestsTool.getModulesToConstraints; +import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput; +import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.getModule; import static net.consensys.linea.ReferenceTestWatcher.JSON_INPUT_FILENAME; import static org.assertj.core.api.Assertions.assertThat; @@ -36,7 +35,6 @@ import net.consensys.linea.corset.CorsetValidator; import net.consensys.linea.testing.ExecutionEnvironment; import net.consensys.linea.zktracer.ZkTracer; -import net.consensys.linea.zktracer.json.JsonConverter; import org.hyperledger.besu.ethereum.MainnetBlockValidator; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -107,14 +105,15 @@ public static CompletableFuture> getRecordedFailedTestsFromJson( if (failedModule.isEmpty()) { return CompletableFuture.completedFuture(failedTests); } - JsonConverter jsonConverter = JsonConverter.builder().build(); - CompletableFuture> modulesToConstraintsFutures = - readFailedTestsOutput(JSON_INPUT_FILENAME) - .thenApply(jsonString -> getModulesToConstraints(jsonString, jsonConverter)); + + CompletableFuture modulesToConstraintsFutures = + readBlockchainReferenceTestsOutput(JSON_INPUT_FILENAME) + .thenApply(ReferenceTestOutcomeRecorderTool::getBlockchainReferenceTestOutcome); return modulesToConstraintsFutures.thenApply( - modulesToConstraints -> { - ModuleToConstraints filteredFailedTests = getModule(modulesToConstraints, failedModule); + blockchainReferenceTestOutcome -> { + ModuleToConstraints filteredFailedTests = + getModule(blockchainReferenceTestOutcome.modulesToConstraints(), failedModule); if (filteredFailedTests == null) { return failedTests; } diff --git a/reference-tests/src/test/java/net/consensys/linea/MapFailedReferenceTestsToolTest.java b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java similarity index 81% rename from reference-tests/src/test/java/net/consensys/linea/MapFailedReferenceTestsToolTest.java rename to reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java index f91d08cdaf..1b375cb13d 100644 --- a/reference-tests/src/test/java/net/consensys/linea/MapFailedReferenceTestsToolTest.java +++ b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestOutcomeRecorderToolTest.java @@ -14,9 +14,9 @@ */ package net.consensys.linea; -import static net.consensys.linea.FailedTestJson.readFailedTestsOutput; -import static net.consensys.linea.MapFailedReferenceTestsTool.getModulesToConstraints; -import static net.consensys.linea.MapFailedReferenceTestsTool.mapAndStoreFailedReferenceTest; +import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput; +import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.getBlockchainReferenceTestOutcome; +import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.mapAndStoreFailedReferenceTest; import static org.assertj.core.api.Assertions.assertThat; import java.io.File; @@ -25,14 +25,12 @@ import java.util.Set; import java.util.stream.Collectors; -import net.consensys.linea.zktracer.json.JsonConverter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -public class MapFailedReferenceTestsToolTest { +public class ReferenceTestOutcomeRecorderToolTest { public static final String TEST_OUTPUT_JSON = "MapFailedReferenceTestsToolTest.json"; - JsonConverter jsonConverter = JsonConverter.builder().build(); @BeforeEach void setup() { @@ -59,11 +57,14 @@ void multipleModulesAreStoredCorrectly() { mapAndStoreFailedReferenceTest(testName, List.of(dummyEvent), TEST_OUTPUT_JSON); - readFailedTestsOutput(TEST_OUTPUT_JSON) + readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON) .thenApply( jsonString -> { + BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = + getBlockchainReferenceTestOutcome(jsonString); + List modulesToConstraints = - getModulesToConstraints(jsonString, jsonConverter); + blockchainReferenceTestOutcome.modulesToConstraints(); assertThat(modulesToConstraints.size()).isEqualTo(modules.size()); assertThat(modulesToConstraints.get(0).moduleName()).isEqualTo(module1); @@ -87,11 +88,14 @@ void multipleConstraintsInSameModuleAreStoredCorrectly() { mapAndStoreFailedReferenceTest(testName, List.of(dummyEvent), TEST_OUTPUT_JSON); - readFailedTestsOutput(TEST_OUTPUT_JSON) + readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON) .thenCompose( jsonString -> { + BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = + getBlockchainReferenceTestOutcome(jsonString); + List modulesToConstraints = - getModulesToConstraints(jsonString, jsonConverter); + blockchainReferenceTestOutcome.modulesToConstraints(); assertThat(modulesToConstraints.size()).isEqualTo(1); @@ -125,11 +129,14 @@ void multipleFailedTestsWithSameConstraintAreStoredCorrectly() { mapAndStoreFailedReferenceTest(testName3, List.of(dummyEvent), TEST_OUTPUT_JSON); mapAndStoreFailedReferenceTest(testName3, List.of(dummyEvent), TEST_OUTPUT_JSON); - readFailedTestsOutput(TEST_OUTPUT_JSON) + readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON) .thenCompose( jsonString -> { + BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = + getBlockchainReferenceTestOutcome(jsonString); + List modulesToConstraints = - getModulesToConstraints(jsonString, jsonConverter); + blockchainReferenceTestOutcome.modulesToConstraints(); assertThat(modulesToConstraints.size()).isEqualTo(modules.size()); Set failedTests = diff --git a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java index 5a75f6fe20..1691257f00 100644 --- a/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java +++ b/reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java @@ -14,6 +14,7 @@ */ package net.consensys.linea; +import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.incrementSuccessRate; import static org.junit.jupiter.api.Assumptions.abort; import java.time.Duration; @@ -49,10 +50,15 @@ public void testFailed(ExtensionContext context, Throwable cause) { List logEventMessages = listAppender.list.stream().map(ILoggingEvent::getMessage).toList(); - MapFailedReferenceTestsTool.mapAndStoreFailedReferenceTest( + ReferenceTestOutcomeRecorderTool.mapAndStoreFailedReferenceTest( testName, logEventMessages, JSON_OUTPUT_FILENAME); } + @Override + public void testSuccessful(ExtensionContext context) { + incrementSuccessRate(JSON_OUTPUT_FILENAME); + } + @Synchronized private static Logger getLogbackLogger() { try {