Keep track of time budget spent fuzzing while updating corpus database.

This is so that we don't restart from the beginning after the workflow gets
preempted and restarted.

#Centipede

PiperOrigin-RevId: 698409531

centipede/centipede_interface.cc

@@ -36,6 +36,7 @@
 #include "absl/log/check.h"
 #include "absl/log/log.h"
 #include "absl/status/status.h"
+#include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -458,6 +459,29 @@ SeedCorpusConfig GetSeedCorpusConfig(const Environment &env,
   };
 }
 
+absl::Duration ReadFuzzingTime(std::string_view fuzzing_time_file) {
+  std::string fuzzing_time_str;
+  CHECK_OK(RemoteFileGetContents(fuzzing_time_file, fuzzing_time_str));
+  absl::Duration fuzzing_time;
+  CHECK(absl::ParseDuration(absl::StripAsciiWhitespace(fuzzing_time_str),
+                            &fuzzing_time))
+      << "Failed to parse fuzzing time of a resuming fuzz test: '"
+      << fuzzing_time_str << "'";
+  return fuzzing_time;
+}
+
+PeriodicAction RecordFuzzingTime(std::string_view fuzzing_time_file,
+                                 absl::Time start_time) {
+  return {[=] {
+            absl::Status status = RemoteFileSetContents(
+                fuzzing_time_file,
+                absl::FormatDuration(absl::Now() - start_time));
+            LOG_IF(WARNING, !status.ok())
+                << "Failed to write fuzzing time: " << status;
+          },
+          PeriodicAction::ZeroDelayConstInterval(absl::Seconds(15))};
+}
+
 // TODO(b/368325638): Add tests for this.
 int UpdateCorpusDatabaseForFuzzTests(
     Environment env, const fuzztest::internal::Configuration &fuzztest_config,
@@ -550,19 +574,33 @@ int UpdateCorpusDatabaseForFuzzTests(
           GetSeedCorpusConfig(env, coverage_dir.c_str()), env.binary_name,
           env.binary_hash));
     }
-    is_resuming = false;
 
     // TODO: b/338217594 - Call the FuzzTest binary in a flag-agnostic way.
     constexpr std::string_view kFuzzTestFuzzFlag = "--fuzz=";
     env.binary = absl::StrCat(binary, " ", kFuzzTestFuzzFlag,
                               fuzztest_config.fuzz_tests[i]);
 
-    LOG(INFO) << "Fuzzing " << fuzztest_config.fuzz_tests[i]
-              << "\n\tTest binary: " << env.binary;
+    absl::Duration time_limit = fuzztest_config.GetTimeLimitPerTest();
+    absl::Duration time_spent = absl::ZeroDuration();
+    const std::string fuzzing_time_file =
+        std::filesystem::path(env.workdir) / "fuzzing_time";
+    if (is_resuming && RemotePathExists(fuzzing_time_file)) {
+      time_spent = ReadFuzzingTime(fuzzing_time_file);
+      time_limit = std::max(time_limit - time_spent, absl::ZeroDuration());
+    }
+    is_resuming = false;
 
-    ClearEarlyStopRequestAndSetStopTime(/*stop_time=*/absl::Now() +
-                                        fuzztest_config.GetTimeLimitPerTest());
+    LOG(INFO) << "Fuzzing " << fuzztest_config.fuzz_tests[i] << " for "
+              << time_limit << "\n\tTest binary: " << env.binary;
+
+    const absl::Time start_time = absl::Now();
+    ClearEarlyStopRequestAndSetStopTime(/*stop_time=*/start_time + time_limit);
+    PeriodicAction record_fuzzing_time =
+        RecordFuzzingTime(fuzzing_time_file, start_time - time_spent);
     Fuzz(env, binary_info, pcs_file_path, callbacks_factory);
+    record_fuzzing_time.Nudge();
+    record_fuzzing_time.Stop();
+
     if (!stats_root_path.empty()) {
       const auto stats_dir = stats_root_path / fuzztest_config.fuzz_tests[i];
       CHECK_OK(RemoteMkdir(stats_dir.c_str()));

e2e_tests/BUILD

@@ -118,8 +118,13 @@ cc_test(
         ":test_binary_util",
         "@com_google_absl//absl/base:no_destructor",
         "@com_google_absl//absl/log:check",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/time",
         "@com_google_fuzztest//centipede:weak_sancov_stubs",
+        "@com_google_fuzztest//fuzztest:io",
+        "@com_google_fuzztest//fuzztest:subprocess",
         "@com_google_googletest//:gtest_main",
     ],
 )

e2e_tests/corpus_database_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <csignal>
 #include <filesystem>  // NOLINT
 #include <string>
 #include <utility>
@@ -20,16 +21,37 @@
 #include "gtest/gtest.h"
 #include "absl/base/no_destructor.h"
 #include "absl/log/check.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
+#include "absl/time/time.h"
 #include "./e2e_tests/test_binary_util.h"
+#include "./fuzztest/internal/io.h"
+#include "./fuzztest/internal/subprocess.h"
 
 namespace fuzztest::internal {
 namespace {
 
 using ::testing::ContainsRegex;
+using ::testing::Eq;
 using ::testing::HasSubstr;
 
+std::string GetCorpusDatabaseTestingBinaryPath() {
+  return BinaryPath((std::filesystem::path("testdata") /
+                     "fuzz_tests_for_corpus_database_testing")
+                        .c_str());
+}
+
+absl::StatusOr<std::string> FindFile(absl::string_view root_path,
+                                     absl::string_view file_name) {
+  for (const std::string &path : ListDirectoryRecursively(root_path)) {
+    if (std::filesystem::path(path).filename() == file_name) return path;
+  }
+  return absl::NotFoundError(absl::StrCat("File ", file_name, " not found."));
+}
+
 class UpdateCorpusDatabaseTest : public testing::Test {
  protected:
   static void SetUpTestSuite() {
@@ -48,16 +70,13 @@ class UpdateCorpusDatabaseTest : public testing::Test {
 
     auto [status, std_out, std_err] = RunBinary(
         CentipedePath(),
-        {.flags = {
-             {"binary",
-              absl::StrCat(BinaryPath((std::filesystem::path("testdata") /
-                                       "fuzz_tests_for_corpus_database_testing")
-                                          .c_str()),
-                           " ",
-                           CreateFuzzTestFlag("corpus_database",
-                                              GetCorpusDatabasePath()),
-                           " ", CreateFuzzTestFlag("fuzz_for", "30s"), " ",
-                           CreateFuzzTestFlag("jobs", "2"))}}});
+        {.flags = {{"binary",
+                    absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(),
+                                   CreateFuzzTestFlag("corpus_database",
+                                                      GetCorpusDatabasePath()),
+                                   CreateFuzzTestFlag("fuzz_for", "30s"),
+                                   CreateFuzzTestFlag("jobs", "2")},
+                                  /*separator=*/" ")}}});
 
     *centipede_std_out_ = std::move(std_out);
     *centipede_std_err_ = std::move(std_err);
@@ -109,5 +128,49 @@ TEST_F(UpdateCorpusDatabaseTest, FindsAllCrashes) {
             ContainsRegex(R"re(Failure\s*: stack-limit-exceeded)re")));
 }
 
+TEST_F(UpdateCorpusDatabaseTest, ResumedFuzzTestRunsForRemainingTime) {
+  TempDir corpus_database;
+
+  // 1st run that gets interrupted.
+  auto [fst_status, fst_std_out, fst_std_err] = RunBinary(
+      CentipedePath(),
+      {.flags = {{"binary",
+                  absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(),
+                                 CreateFuzzTestFlag("corpus_database",
+                                                    corpus_database.dirname()),
+                                 CreateFuzzTestFlag("fuzz_for", "300s")},
+                                /*separator=*/" ")},
+                 // Disable symbolization to more quickly get to fuzzing.
+                 {"symbolizer_path", ""}},
+       // Stop the binary with SIGTERM before the fuzzing is done.
+       .timeout = absl::Seconds(10)});
+  ASSERT_THAT(fst_status, Eq(Signal(SIGTERM)));
+
+  // Adjust the fuzzing time so that only 1s remains.
+  const absl::StatusOr<std::string> fuzzing_time_file =
+      FindFile(corpus_database.dirname(), "fuzzing_time");
+  ASSERT_TRUE(fuzzing_time_file.ok());
+  ASSERT_TRUE(WriteFile(*fuzzing_time_file, "299s"));
+
+  // 2nd run that resumes the fuzzing.
+  auto [snd_status, snd_std_out, snd_std_err] = RunBinary(
+      CentipedePath(),
+      {.flags = {{"binary",
+                  absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(),
+                                 CreateFuzzTestFlag("corpus_database",
+                                                    corpus_database.dirname()),
+                                 CreateFuzzTestFlag("fuzz_for", "300s")},
+                                /*separator=*/" ")},
+                 // Disable symbolization to more quickly get to fuzzing.
+                 {"symbolizer_path", ""}},
+       .timeout = absl::Seconds(10)});
+
+  EXPECT_THAT(
+      snd_std_err,
+      // The resumed fuzz test is the first one defined in the binary.
+      AllOf(HasSubstr("Resuming from the fuzz test FuzzTest.FailsInTwoWays"),
+            HasSubstr("Fuzzing FuzzTest.FailsInTwoWays for 1s")));
+}
+
 }  // namespace
 }  // namespace fuzztest::internal