Skip to content

Commit

Permalink
Blockchain reference tests report - success and failure counters (#1311)
Browse files Browse the repository at this point in the history 
* New object for reading and writing success and failure counters to json

* Spotless
  • Loading branch information
@bradbown
bradbown authored Sep 25, 2024
1 parent 9c15018 commit ae20d75
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 49 deletions.
4 changes: 2 additions & 2 deletions ...a/net/consensys/linea/FailedTestJson.java → ...ys/linea/BlockchainReferenceTestJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class FailedTestJson {
public class BlockchainReferenceTestJson {
static String fileDirectory = setFileDirectory();

@Synchronized
public static CompletableFuture<String> readFailedTestsOutput(String fileName) {
public static CompletableFuture<String> readBlockchainReferenceTestsOutput(String fileName) {
return CompletableFuture.supplyAsync(
() -> {
Path directoryPath = Paths.get(fileDirectory);
Expand Down
23 changes: 23 additions & 0 deletions reference-tests/src/main/java/net/consensys/linea/BlockchainReferenceTestOutcome.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright Consensys Software Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package net.consensys.linea;

import java.util.List;

import com.fasterxml.jackson.annotation.JsonPropertyOrder;

@JsonPropertyOrder({"failedCounter", "successCounter", "modules"})
public record BlockchainReferenceTestOutcome(
int failedCounter, int successCounter, List<ModuleToConstraints> modulesToConstraints) {}
8 changes: 0 additions & 8 deletions reference-tests/src/main/java/net/consensys/linea/ModuleToConstraints.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,9 @@
import java.util.stream.Collectors;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;

public record ModuleToConstraints(String moduleName, Map<String, Set<String>> constraints) {

public ModuleToConstraints(
@JsonProperty("moduleName") String moduleName,
@JsonProperty("constraints") Map<String, Set<String>> constraints) {
this.moduleName = moduleName;
this.constraints = constraints;
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand Down
60 changes: 44 additions & 16 deletions ...ys/linea/MapFailedReferenceTestsTool.java → ...nea/ReferenceTestOutcomeRecorderTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
*/
package net.consensys.linea;

import static net.consensys.linea.FailedTestJson.readFailedTestsOutput;
import static net.consensys.linea.FailedTestJson.writeToJsonFile;
import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput;
import static net.consensys.linea.BlockchainReferenceTestJson.writeToJsonFile;

import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -33,31 +33,58 @@
import net.consensys.linea.zktracer.json.JsonConverter;

@Slf4j
public class MapFailedReferenceTestsTool {
public class ReferenceTestOutcomeRecorderTool {

static JsonConverter jsonConverter = JsonConverter.builder().build();

@Synchronized
public static void mapAndStoreFailedReferenceTest(
String testName, List<String> logEventMessages, String jsonOutputFilename) {
Set<String> failedConstraints = getFailedConstraints(logEventMessages);
if (!failedConstraints.isEmpty()) {
readFailedTestsOutput(jsonOutputFilename)
readBlockchainReferenceTestsOutput(jsonOutputFilename)
.thenCompose(
jsonString -> {
JsonConverter jsonConverter = JsonConverter.builder().build();

List<ModuleToConstraints> modulesToConstraints =
getModulesToConstraints(jsonString, jsonConverter);
BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
getBlockchainReferenceTestOutcome(jsonString);

mapFailedConstraintsToTestsToModule(
modulesToConstraints, failedConstraints, testName);

jsonString = jsonConverter.toJson(modulesToConstraints);
blockchainReferenceTestOutcome.modulesToConstraints(),
failedConstraints,
testName);

BlockchainReferenceTestOutcome finalBlockchainReferenceTestOutcome =
new BlockchainReferenceTestOutcome(
blockchainReferenceTestOutcome.failedCounter() + 1,
blockchainReferenceTestOutcome.successCounter(),
blockchainReferenceTestOutcome.modulesToConstraints());
jsonString = jsonConverter.toJson(finalBlockchainReferenceTestOutcome);
writeToJsonFile(jsonString, jsonOutputFilename);
return null;
});
}
}

@Synchronized
public static void incrementSuccessRate(String jsonOutputFilename) {
readBlockchainReferenceTestsOutput(jsonOutputFilename)
.thenAccept(
jsonString -> {
BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
getBlockchainReferenceTestOutcome(jsonString);

BlockchainReferenceTestOutcome finalBlockchainReferenceTestOutcome =
new BlockchainReferenceTestOutcome(
blockchainReferenceTestOutcome.failedCounter(),
blockchainReferenceTestOutcome.successCounter() + 1,
blockchainReferenceTestOutcome.modulesToConstraints());
jsonString = jsonConverter.toJson(finalBlockchainReferenceTestOutcome);
writeToJsonFile(jsonString, jsonOutputFilename);
});
}

@Synchronized
private static void mapFailedConstraintsToTestsToModule(
List<ModuleToConstraints> modulesToConstraints,
Expand Down Expand Up @@ -119,15 +146,16 @@ private static void addModuleIfAbsent(
}

@Synchronized
public static List<ModuleToConstraints> getModulesToConstraints(
String jsonString, JsonConverter jsonConverter) {
List<ModuleToConstraints> moduleToConstraints = new ArrayList<>();
public static BlockchainReferenceTestOutcome getBlockchainReferenceTestOutcome(
String jsonString) {
BlockchainReferenceTestOutcome blockchainReferenceTestOutcome = null;
if (!jsonString.isEmpty()) {
moduleToConstraints =
new ArrayList<>(
Arrays.asList(jsonConverter.fromJson(jsonString, ModuleToConstraints[].class)));
blockchainReferenceTestOutcome =
jsonConverter.fromJson(jsonString, BlockchainReferenceTestOutcome.class);
} else {
blockchainReferenceTestOutcome = new BlockchainReferenceTestOutcome(0, 0, new ArrayList<>());
}
return moduleToConstraints;
return blockchainReferenceTestOutcome;
}

private static Set<String> getFailedConstraints(List<String> logEventMessages) {
Expand Down
19 changes: 9 additions & 10 deletions reference-tests/src/test/java/net/consensys/linea/BlockchainReferenceTestTools.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@

package net.consensys.linea;

import static net.consensys.linea.FailedTestJson.readFailedTestsOutput;
import static net.consensys.linea.MapFailedReferenceTestsTool.getModule;
import static net.consensys.linea.MapFailedReferenceTestsTool.getModulesToConstraints;
import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput;
import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.getModule;
import static net.consensys.linea.ReferenceTestWatcher.JSON_INPUT_FILENAME;
import static org.assertj.core.api.Assertions.assertThat;

Expand All @@ -36,7 +35,6 @@
import net.consensys.linea.corset.CorsetValidator;
import net.consensys.linea.testing.ExecutionEnvironment;
import net.consensys.linea.zktracer.ZkTracer;
import net.consensys.linea.zktracer.json.JsonConverter;
import org.hyperledger.besu.ethereum.MainnetBlockValidator;
import org.hyperledger.besu.ethereum.ProtocolContext;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
Expand Down Expand Up @@ -107,14 +105,15 @@ public static CompletableFuture<Set<String>> getRecordedFailedTestsFromJson(
if (failedModule.isEmpty()) {
return CompletableFuture.completedFuture(failedTests);
}
JsonConverter jsonConverter = JsonConverter.builder().build();
CompletableFuture<List<ModuleToConstraints>> modulesToConstraintsFutures =
readFailedTestsOutput(JSON_INPUT_FILENAME)
.thenApply(jsonString -> getModulesToConstraints(jsonString, jsonConverter));

CompletableFuture<BlockchainReferenceTestOutcome> modulesToConstraintsFutures =
readBlockchainReferenceTestsOutput(JSON_INPUT_FILENAME)
.thenApply(ReferenceTestOutcomeRecorderTool::getBlockchainReferenceTestOutcome);

return modulesToConstraintsFutures.thenApply(
modulesToConstraints -> {
ModuleToConstraints filteredFailedTests = getModule(modulesToConstraints, failedModule);
blockchainReferenceTestOutcome -> {
ModuleToConstraints filteredFailedTests =
getModule(blockchainReferenceTestOutcome.modulesToConstraints(), failedModule);
if (filteredFailedTests == null) {
return failedTests;
}
Expand Down
31 changes: 19 additions & 12 deletions ...inea/MapFailedReferenceTestsToolTest.java → ...ReferenceTestOutcomeRecorderToolTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
*/
package net.consensys.linea;

import static net.consensys.linea.FailedTestJson.readFailedTestsOutput;
import static net.consensys.linea.MapFailedReferenceTestsTool.getModulesToConstraints;
import static net.consensys.linea.MapFailedReferenceTestsTool.mapAndStoreFailedReferenceTest;
import static net.consensys.linea.BlockchainReferenceTestJson.readBlockchainReferenceTestsOutput;
import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.getBlockchainReferenceTestOutcome;
import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.mapAndStoreFailedReferenceTest;
import static org.assertj.core.api.Assertions.assertThat;

import java.io.File;
Expand All @@ -25,14 +25,12 @@
import java.util.Set;
import java.util.stream.Collectors;

import net.consensys.linea.zktracer.json.JsonConverter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

public class MapFailedReferenceTestsToolTest {
public class ReferenceTestOutcomeRecorderToolTest {

public static final String TEST_OUTPUT_JSON = "MapFailedReferenceTestsToolTest.json";
JsonConverter jsonConverter = JsonConverter.builder().build();

@BeforeEach
void setup() {
Expand All @@ -59,11 +57,14 @@ void multipleModulesAreStoredCorrectly() {

mapAndStoreFailedReferenceTest(testName, List.of(dummyEvent), TEST_OUTPUT_JSON);

readFailedTestsOutput(TEST_OUTPUT_JSON)
readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON)
.thenApply(
jsonString -> {
BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
getBlockchainReferenceTestOutcome(jsonString);

List<ModuleToConstraints> modulesToConstraints =
getModulesToConstraints(jsonString, jsonConverter);
blockchainReferenceTestOutcome.modulesToConstraints();

assertThat(modulesToConstraints.size()).isEqualTo(modules.size());
assertThat(modulesToConstraints.get(0).moduleName()).isEqualTo(module1);
Expand All @@ -87,11 +88,14 @@ void multipleConstraintsInSameModuleAreStoredCorrectly() {

mapAndStoreFailedReferenceTest(testName, List.of(dummyEvent), TEST_OUTPUT_JSON);

readFailedTestsOutput(TEST_OUTPUT_JSON)
readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON)
.thenCompose(
jsonString -> {
BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
getBlockchainReferenceTestOutcome(jsonString);

List<ModuleToConstraints> modulesToConstraints =
getModulesToConstraints(jsonString, jsonConverter);
blockchainReferenceTestOutcome.modulesToConstraints();

assertThat(modulesToConstraints.size()).isEqualTo(1);

Expand Down Expand Up @@ -125,11 +129,14 @@ void multipleFailedTestsWithSameConstraintAreStoredCorrectly() {
mapAndStoreFailedReferenceTest(testName3, List.of(dummyEvent), TEST_OUTPUT_JSON);
mapAndStoreFailedReferenceTest(testName3, List.of(dummyEvent), TEST_OUTPUT_JSON);

readFailedTestsOutput(TEST_OUTPUT_JSON)
readBlockchainReferenceTestsOutput(TEST_OUTPUT_JSON)
.thenCompose(
jsonString -> {
BlockchainReferenceTestOutcome blockchainReferenceTestOutcome =
getBlockchainReferenceTestOutcome(jsonString);

List<ModuleToConstraints> modulesToConstraints =
getModulesToConstraints(jsonString, jsonConverter);
blockchainReferenceTestOutcome.modulesToConstraints();

assertThat(modulesToConstraints.size()).isEqualTo(modules.size());
Set<String> failedTests =
Expand Down
8 changes: 7 additions & 1 deletion reference-tests/src/test/java/net/consensys/linea/ReferenceTestWatcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*/
package net.consensys.linea;

import static net.consensys.linea.ReferenceTestOutcomeRecorderTool.incrementSuccessRate;
import static org.junit.jupiter.api.Assumptions.abort;

import java.time.Duration;
Expand Down Expand Up @@ -49,10 +50,15 @@ public void testFailed(ExtensionContext context, Throwable cause) {
List<String> logEventMessages =
listAppender.list.stream().map(ILoggingEvent::getMessage).toList();

MapFailedReferenceTestsTool.mapAndStoreFailedReferenceTest(
ReferenceTestOutcomeRecorderTool.mapAndStoreFailedReferenceTest(
testName, logEventMessages, JSON_OUTPUT_FILENAME);
}

@Override
public void testSuccessful(ExtensionContext context) {
incrementSuccessRate(JSON_OUTPUT_FILENAME);
}

@Synchronized
private static Logger getLogbackLogger() {
try {
Expand Down

0 comments on commit ae20d75

Please sign in to comment.