diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java index 8162bb368..45d15baef 100644 --- a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/junit/quickcheck/FuzzStatement.java @@ -30,21 +30,12 @@ package edu.berkeley.cs.jqf.fuzz.junit.quickcheck; import java.io.EOFException; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; import java.lang.reflect.Parameter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.pholser.junit.quickcheck.generator.GenerationStatus; import com.pholser.junit.quickcheck.generator.Generator; import com.pholser.junit.quickcheck.internal.ParameterTypeContext; @@ -127,7 +118,7 @@ public void evaluate() throws Throwable { long startTrialTime = System.currentTimeMillis(); // Initialize guided fuzzing using a file-backed random number source - Object [] args = null; + Object [] args = {}; try { try { @@ -191,17 +182,12 @@ public void evaluate() throws Throwable { } long endTrialTime = System.currentTimeMillis(); if (System.getProperty("jqfObservability") != null) { - - - // - "status": "passed", "failed", or "gave_up" observability.addStatus(result); if (result == SUCCESS) { observability.addTiming(startTrialTime, endGenerationTime, endTrialTime); } observability.add("representation", Arrays.toString(args)); - // - "status_reason": If non-empty, the reason for which the test failed or was abandoned. - // - "how_generated": "Zest", "blind", or "repro" if (guidance instanceof ZestGuidance) { observability.add("how_generated", "Zest"); } else if (guidance instanceof NoGuidance) { @@ -213,8 +199,6 @@ public void evaluate() throws Throwable { } observability.writeToFile(); - - } // Inform guidance about the outcome of this trial diff --git a/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java new file mode 100644 index 000000000..a8977948a --- /dev/null +++ b/fuzz/src/main/java/edu/berkeley/cs/jqf/fuzz/util/Observability.java @@ -0,0 +1,100 @@ +package edu.berkeley.cs.jqf.fuzz.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.berkeley.cs.jqf.fuzz.guidance.Result; + +import java.io.FileWriter; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static edu.berkeley.cs.jqf.fuzz.guidance.Result.FAILURE; +import static edu.berkeley.cs.jqf.fuzz.guidance.Result.INVALID; + +public class Observability { + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final String testClass; + private final String testMethod; + private final Path obsPath; + private static ObjectNode testCaseJsonObject; + private final long startTime; + + public Observability(String testClass, String testMethod, long startTime) { + this.testClass = testClass; + this.testMethod = testMethod; + this.obsPath = Paths.get("target", "fuzz-results", testClass, testMethod, "observations.jsonl"); + if (obsPath.toFile().exists()) { + obsPath.toFile().delete(); + } + this.startTime = startTime; + this.initializeTestCase(); + } + + public void initializeTestCase() { + testCaseJsonObject = objectMapper.createObjectNode(); + testCaseJsonObject.putObject("features"); + testCaseJsonObject.putObject("timing"); + testCaseJsonObject.putObject("coverage"); + testCaseJsonObject.putObject("args"); + testCaseJsonObject.putObject("metadata"); + testCaseJsonObject.put("type", "test_case"); + testCaseJsonObject.put("run_start", startTime); + testCaseJsonObject.put("property", testMethod); + } + + public static void event(String value, Object payload) throws RuntimeException { + // Add the payload to the features object + JsonNode jsonFeaturesNode = testCaseJsonObject.get("features"); + ObjectNode featuresNode = (ObjectNode) jsonFeaturesNode; + + if (payload instanceof Integer) { + featuresNode.put(value, (Integer) payload); + } else if (payload instanceof String) { + featuresNode.put(value, (String) payload); + } else if (payload instanceof Float) { + featuresNode.put(value, (Float) payload); + } else { + throw new RuntimeException("Unsupported payload type for event"); + } + } + + public void addStatus(Result result) { + if (result == INVALID) { + testCaseJsonObject.put("status", "gave_up"); + testCaseJsonObject.put("status_reason", "assumption violated"); + } else if (result == FAILURE) { + testCaseJsonObject.put("status", "failed"); + testCaseJsonObject.put("status_reason", "Encountered exception"); + } else { + testCaseJsonObject.put("status", "passed"); + testCaseJsonObject.put("status_reason", ""); + } + + } + + public void addTiming(long startTime, long endGenerationTime, long endExecutionTime) { + JsonNode timingNode = testCaseJsonObject.get("timing"); + ObjectNode timingObject = (ObjectNode) timingNode; + timingObject.put("generation", endGenerationTime - startTime); + timingObject.put("execution", endExecutionTime - endGenerationTime); + } + + public void add(String key, String value) { + testCaseJsonObject.put(key, value); + } + + public void writeToFile() { + // Append the JSON object to a file followed by a newline + try { + String jsonString = objectMapper.writeValueAsString(testCaseJsonObject); + try (FileWriter writer = new FileWriter(obsPath.toFile(), true)) { + writer.write(jsonString); + writer.write(System.lineSeparator()); // Add a new line after each object + } + } catch (Exception e) { + throw new RuntimeException("Failed to write observations to file", e); + } + } +}