From ba554ec8ac5721e1380528e3d3df873edffbaa47 Mon Sep 17 00:00:00 2001 From: Xinhao Yuan Date: Wed, 18 Dec 2024 09:22:42 -0800 Subject: [PATCH] No public description PiperOrigin-RevId: 707574993 --- centipede/centipede_callbacks.cc | 5 + centipede/centipede_interface.cc | 138 +++++---- centipede/environment.cc | 2 +- centipede/environment.h | 3 + centipede/environment_test.cc | 2 +- e2e_tests/corpus_database_test.cc | 161 ++++++---- e2e_tests/functional_test.cc | 178 ++++++----- fuzztest/BUILD | 4 +- fuzztest/init_fuzztest.cc | 96 ++++-- fuzztest/internal/centipede_adaptor.cc | 386 +++++++++++++----------- fuzztest/internal/centipede_adaptor.h | 9 +- fuzztest/internal/configuration.cc | 15 +- fuzztest/internal/configuration.h | 8 +- fuzztest/internal/configuration_test.cc | 12 +- fuzztest/internal/googletest_adaptor.h | 7 +- fuzztest/internal/runtime.cc | 22 +- fuzztest/internal/runtime.h | 15 +- 17 files changed, 632 insertions(+), 431 deletions(-) diff --git a/centipede/centipede_callbacks.cc b/centipede/centipede_callbacks.cc index 436e6d24..895c0117 100644 --- a/centipede/centipede_callbacks.cc +++ b/centipede/centipede_callbacks.cc @@ -264,6 +264,11 @@ bool CentipedeCallbacks::GetSeedsViaExternalBinary( .temp_file_path = temp_input_file_path_}}; const int retval = cmd.Execute(); + if (env_.print_runner_log) { + LOG(INFO) << "Getting seeds via external binary returns " << retval; + PrintExecutionLog(); + } + std::vector seed_input_filenames; for (const auto &dir_ent : std::filesystem::directory_iterator(output_dir)) { seed_input_filenames.push_back(dir_ent.path().filename()); diff --git a/centipede/centipede_interface.cc b/centipede/centipede_interface.cc index b21cdd92..692fd2b4 100644 --- a/centipede/centipede_interface.cc +++ b/centipede/centipede_interface.cc @@ -431,22 +431,34 @@ void DeduplicateAndStoreNewCrashes( } } -// Seeds the corpus files in `env.workdir` with the previously distilled corpus -// files from `src_dir`. +// Seeds the corpus files in `env.workdir` with the inputs in `regression_dir` +// (always used) and the previously distilled corpus files from `coverage_dir` +// (used if non-empty). SeedCorpusConfig GetSeedCorpusConfig(const Environment &env, - std::string_view src_dir) { + std::string_view regression_dir, + std::string_view coverage_dir) { const WorkDir workdir{env}; + std::vector sources = {{ + .dir_glob = std::string(regression_dir), + .num_recent_dirs = 1, + .individual_input_rel_glob = "*", + .sampled_fraction_or_count = 1.0f, + }}; + if (!coverage_dir.empty()) { + sources.push_back(SeedCorpusSource{ + .dir_glob = std::string(coverage_dir), + .num_recent_dirs = 1, + // We're using the previously distilled corpus files as seeds. + .shard_rel_glob = + std::filesystem::path{ + workdir.DistilledCorpusFilePaths().AllShardsGlob()} + .filename(), + .individual_input_rel_glob = "*", + .sampled_fraction_or_count = 1.0f, + }); + } return { - .sources = {SeedCorpusSource{ - .dir_glob = std::string(src_dir), - .num_recent_dirs = 1, - // We're using the previously distilled corpus files as seeds. - .shard_rel_glob = - std::filesystem::path{ - workdir.DistilledCorpusFilePaths().AllShardsGlob()} - .filename(), - .sampled_fraction_or_count = 1.0f, - }}, + .sources = std::move(sources), .destination = { .dir_path = env.workdir, @@ -495,9 +507,7 @@ int UpdateCorpusDatabaseForFuzzTests( absl::Time start_time = absl::Now(); LOG(INFO) << "Starting the update of the corpus database for fuzz tests:" << "\nBinary: " << env.binary - << "\nCorpus database: " << fuzztest_config.corpus_database - << "\nFuzz tests: " - << absl::StrJoin(fuzztest_config.fuzz_tests, ", "); + << "\nCorpus database: " << fuzztest_config.corpus_database; // Step 1: Preliminary set up of test sharding, binary info, etc. const auto [test_shard_index, total_test_shards] = SetUpTestSharding(); @@ -514,14 +524,32 @@ int UpdateCorpusDatabaseForFuzzTests( absl::FormatTime("%Y-%m-%d-%H-%M-%S", absl::Now(), absl::UTCTimeZone()); return stamp; }(); + std::vector fuzz_tests_to_run; + if (env.fuzztest_single_test_mode) { + CHECK(fuzztest_config.fuzz_tests_in_current_shard.size() == 1) + << "Must select exactly one fuzz test when running in the unified " + "exeuction model."; + fuzz_tests_to_run = fuzztest_config.fuzz_tests_in_current_shard; + } else { + for (int i = 0; i < fuzztest_config.fuzz_tests.size(); ++i) { + if (i % total_test_shards == test_shard_index) { + fuzz_tests_to_run.push_back(fuzztest_config.fuzz_tests[i]); + } + } + } + LOG(INFO) << "Fuzz tests to run:" << absl::StrJoin(fuzz_tests_to_run, ", "); + + const bool is_workdir_specified = !env.workdir.empty(); // The full workdir paths will be formed by appending the fuzz test names to // the base workdir path. We use different path when only replaying to avoid // replaying an unfinished fuzzing sessions. const auto base_workdir_path = - corpus_database_path / - absl::StrFormat("workdir%s.%03d", - fuzztest_config.only_replay_corpus ? "-replay" : "", - test_shard_index); + is_workdir_specified + ? std::filesystem::path(env.workdir) + : corpus_database_path / + absl::StrFormat("workdir%s.%03d", + fuzztest_config.only_replay ? "-replay" : "", + test_shard_index); // There's no point in saving the binary info to the workdir, since the // workdir is deleted at the end. env.save_binary_info = false; @@ -536,9 +564,8 @@ int UpdateCorpusDatabaseForFuzzTests( // Find the last index of a fuzz test for which we already have a workdir. bool is_resuming = false; int resuming_fuzztest_idx = 0; - for (int i = 0; i < fuzztest_config.fuzz_tests.size(); ++i) { - if (i % total_test_shards != test_shard_index) continue; - env.workdir = base_workdir_path / fuzztest_config.fuzz_tests[i]; + for (int i = 0; i < fuzz_tests_to_run.size(); ++i) { + env.workdir = base_workdir_path / fuzz_tests_to_run[i]; // Check the existence of the coverage path to not only make sure the // workdir exists, but also that it was created for the same binary as in // this run. @@ -549,22 +576,20 @@ int UpdateCorpusDatabaseForFuzzTests( } LOG_IF(INFO, is_resuming) << "Resuming from the fuzz test " - << fuzztest_config.fuzz_tests[resuming_fuzztest_idx] + << fuzz_tests_to_run[resuming_fuzztest_idx] << " (index: " << resuming_fuzztest_idx << ")"; // Step 3: Iterate over the fuzz tests and run them. const std::string binary = env.binary; - for (int i = resuming_fuzztest_idx; i < fuzztest_config.fuzz_tests.size(); - ++i) { - if (i % total_test_shards != test_shard_index) continue; - if (fuzztest_config.GetTimeLimitPerTest() < absl::InfiniteDuration()) { - // TODO(fniksic): Test this behavior in end-to-end tests. + for (int i = resuming_fuzztest_idx; i < fuzz_tests_to_run.size(); ++i) { + if (!env.fuzztest_single_test_mode && + fuzztest_config.GetTimeLimitPerTest() < absl::InfiniteDuration()) { ReportErrorWhenNotEnoughTimeToRunEverything( start_time, fuzztest_config.GetTimeLimitPerTest(), /*executed_tests_in_shard=*/i / total_test_shards, fuzztest_config.fuzz_tests.size(), total_test_shards); } - env.workdir = base_workdir_path / fuzztest_config.fuzz_tests[i]; + env.workdir = base_workdir_path / fuzz_tests_to_run[i]; if (RemotePathExists(env.workdir) && !is_resuming) { // This could be a workdir from a failed run that used a different version // of the binary. We delete it so that we don't have to deal with the @@ -575,26 +600,36 @@ int UpdateCorpusDatabaseForFuzzTests( CHECK_OK(RemoteMkdir( workdir.CoverageDirPath())); // Implicitly creates the workdir - // Seed the fuzzing session with the latest coverage corpus from the - // previous fuzzing session. const std::filesystem::path fuzztest_db_path = - corpus_database_path / fuzztest_config.fuzz_tests[i]; + corpus_database_path / fuzz_tests_to_run[i]; + const std::filesystem::path regression_dir = + fuzztest_db_path / "regression"; const std::filesystem::path coverage_dir = fuzztest_db_path / "coverage"; - if (RemotePathExists(coverage_dir.c_str()) && !is_resuming) { - CHECK_OK(GenerateSeedCorpusFromConfig( - GetSeedCorpusConfig(env, coverage_dir.c_str()), env.binary_name, - env.binary_hash)); + + // Seed the fuzzing session with the latest coverage corpus and regression + // inputs from the previous fuzzing session. + if (!is_resuming) { + if (const auto status = GenerateSeedCorpusFromConfig( + GetSeedCorpusConfig( + env, regression_dir.c_str(), + fuzztest_config.replay_corpus ? coverage_dir.c_str() : ""), + env.binary_name, env.binary_hash); + !status.ok()) { + LOG(ERROR) << "Got error while generating the seed corpus: " << status; + } } - // TODO: b/338217594 - Call the FuzzTest binary in a flag-agnostic way. - constexpr std::string_view kFuzzTestFuzzFlag = "--fuzz="; - constexpr std::string_view kFuzzTestReplayCorpusFlag = - "--replay_corpus="; - std::string_view test_selection_flag = fuzztest_config.only_replay_corpus - ? kFuzzTestReplayCorpusFlag - : kFuzzTestFuzzFlag; - env.binary = absl::StrCat(binary, " ", test_selection_flag, - fuzztest_config.fuzz_tests[i]); + if (!env.fuzztest_single_test_mode) { + // TODO: b/338217594 - Call the FuzzTest binary in a flag-agnostic way. + constexpr std::string_view kFuzzTestFuzzFlag = "--fuzz="; + constexpr std::string_view kFuzzTestReplayCorpusFlag = + "--replay_corpus="; + std::string_view test_selection_flag = fuzztest_config.only_replay + ? kFuzzTestReplayCorpusFlag + : kFuzzTestFuzzFlag; + env.binary = + absl::StrCat(binary, " ", test_selection_flag, fuzz_tests_to_run[i]); + } absl::Duration time_limit = fuzztest_config.GetTimeLimitPerTest(); absl::Duration time_spent = absl::ZeroDuration(); @@ -606,9 +641,8 @@ int UpdateCorpusDatabaseForFuzzTests( } is_resuming = false; - LOG(INFO) << (fuzztest_config.only_replay_corpus ? "Replaying " - : "Fuzzing ") - << fuzztest_config.fuzz_tests[i] << " for " << time_limit + LOG(INFO) << (fuzztest_config.only_replay ? "Replaying " : "Fuzzing ") + << fuzz_tests_to_run[i] << " for " << time_limit << "\n\tTest binary: " << env.binary; const absl::Time start_time = absl::Now(); @@ -620,7 +654,7 @@ int UpdateCorpusDatabaseForFuzzTests( record_fuzzing_time.Stop(); if (!stats_root_path.empty()) { - const auto stats_dir = stats_root_path / fuzztest_config.fuzz_tests[i]; + const auto stats_dir = stats_root_path / fuzz_tests_to_run[i]; CHECK_OK(RemoteMkdir(stats_dir.c_str())); CHECK_OK(RemotePathRename( workdir.FuzzingStatsPath(), @@ -628,7 +662,7 @@ int UpdateCorpusDatabaseForFuzzTests( .c_str())); } - if (fuzztest_config.only_replay_corpus) continue; + if (fuzztest_config.only_replay || is_workdir_specified) continue; // Distill and store the coverage corpus. Distill(env); @@ -720,7 +754,7 @@ int CentipedeMain(const Environment &env, << "Failed to deserialize target configuration"; if (!target_config->corpus_database.empty()) { const auto time_limit_per_test = target_config->GetTimeLimitPerTest(); - CHECK(target_config->only_replay_corpus || + CHECK(target_config->only_replay || time_limit_per_test < absl::InfiniteDuration()) << "Updating corpus database requires specifying time limit per " "fuzz test."; diff --git a/centipede/environment.cc b/centipede/environment.cc index 10bd8ee7..a1679dd9 100644 --- a/centipede/environment.cc +++ b/centipede/environment.cc @@ -304,7 +304,7 @@ void Environment::UpdateWithTargetConfig( << VV(stack_limit_kb) << VV(config.stack_limit); stack_limit_kb = bytes_to_kb(config.stack_limit); - if (config.only_replay_corpus) { + if (config.only_replay) { load_shards_only = true; populate_binary_info = false; } diff --git a/centipede/environment.h b/centipede/environment.h index 751fad9e..79a71b10 100644 --- a/centipede/environment.h +++ b/centipede/environment.h @@ -127,6 +127,9 @@ struct Environment { bool first_corpus_dir_output_only = false; // If set, load/merge shards without fuzzing new inputs. bool load_shards_only = false; + // If set, operate on the corpus database for a single test specified by + // FuzzTest instead of all the tests. + bool fuzztest_single_test_mode = false; // Command line-related fields ----------------------------------------------- diff --git a/centipede/environment_test.cc b/centipede/environment_test.cc index 2f788a68..68e5bdca 100644 --- a/centipede/environment_test.cc +++ b/centipede/environment_test.cc @@ -198,7 +198,7 @@ TEST(Environment, DiesOnInconsistentStackLimitKbAndTargetConfigStackLimit) { TEST(Environment, UpdatesReplayOnlyConfiguration) { Environment env; - fuzztest::internal::Configuration config{.only_replay_corpus = true}; + fuzztest::internal::Configuration config{.only_replay = true}; env.UpdateWithTargetConfig(config); EXPECT_TRUE(env.load_shards_only); EXPECT_FALSE(env.populate_binary_info); diff --git a/e2e_tests/corpus_database_test.cc b/e2e_tests/corpus_database_test.cc index d2d356e5..c834e98b 100644 --- a/e2e_tests/corpus_database_test.cc +++ b/e2e_tests/corpus_database_test.cc @@ -13,9 +13,12 @@ // limitations under the License. #include +#include +#include #include // NOLINT #include #include +#include #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -52,7 +55,13 @@ absl::StatusOr FindFile(absl::string_view root_path, return absl::NotFoundError(absl::StrCat("File ", file_name, " not found.")); } -class UpdateCorpusDatabaseTest : public testing::Test { +enum class ExecutionModelParam { + kSingleBinary, + kWithCentipedeBinary, +}; + +class UpdateCorpusDatabaseTest + : public ::testing::TestWithParam { protected: static void SetUpTestSuite() { #if defined(__has_feature) @@ -65,21 +74,21 @@ class UpdateCorpusDatabaseTest : public testing::Test { "Please run with --config=fuzztest-experimental."; #endif #endif + CHECK(temp_dir_ == nullptr); + } + static void RunUpdateCorpusDatabase() { + if (temp_dir_ != nullptr) return; temp_dir_ = new TempDir(); - - auto [status, std_out, std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - GetCorpusDatabasePath()), - CreateFuzzTestFlag("fuzz_for", "30s"), - CreateFuzzTestFlag("jobs", "2")}, - /*separator=*/" ")}}}); - - *centipede_std_out_ = std::move(std_out); - *centipede_std_err_ = std::move(std_err); + auto [status, std_out, std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = { + {"corpus_database", GetCorpusDatabasePath()}, + {"fuzz_for", "30s"}, + {"jobs", "2"}, + }}); + *update_corpus_database_std_out_ = std::move(std_out); + *update_corpus_database_std_err_ = std::move(std_err); } static void TearDownTestSuite() { @@ -88,81 +97,110 @@ class UpdateCorpusDatabaseTest : public testing::Test { } static std::string GetCorpusDatabasePath() { - CHECK(temp_dir_ != nullptr); + RunUpdateCorpusDatabase(); return std::filesystem::path(temp_dir_->dirname()) / "corpus_database"; } - static absl::string_view GetCentipedeStdOut() { return *centipede_std_out_; } + static absl::string_view GetUpdateCorpusDatabaseStdOut() { + RunUpdateCorpusDatabase(); + return *update_corpus_database_std_out_; + } - static absl::string_view GetCentipedeStdErr() { return *centipede_std_err_; } + static absl::string_view GetUpdateCorpusDatabaseStdErr() { + RunUpdateCorpusDatabase(); + return *update_corpus_database_std_err_; + } + + static RunResults RunBinaryMaybeWithCentipede(absl::string_view binary_path, + const RunOptions &options) { + switch (GetParam()) { + case ExecutionModelParam::kSingleBinary: + return RunBinary(binary_path, options); + case ExecutionModelParam::kWithCentipedeBinary: { + RunOptions actual_options; + actual_options.env = options.env; + actual_options.timeout = options.timeout; + std::vector binary_args; + binary_args.push_back(std::string(binary_path)); + for (const auto &[key, value] : options.fuzztest_flags) { + binary_args.push_back(CreateFuzzTestFlag(key, value)); + } + for (const auto &[key, value] : options.flags) { + binary_args.push_back(absl::StrCat("--", key, "=", value)); + } + actual_options.flags = { + {"binary", absl::StrJoin(binary_args, " ")}, + // Disable symbolization to more quickly get to fuzzing. + {"symbolizer_path", ""}, + }; + return RunBinary(CentipedePath(), actual_options); + } + } + fprintf(stderr, "Unsupported execution model!\n"); + std::abort(); + } private: static TempDir *temp_dir_; - static absl::NoDestructor centipede_std_out_; - static absl::NoDestructor centipede_std_err_; + static absl::NoDestructor update_corpus_database_std_out_; + static absl::NoDestructor update_corpus_database_std_err_; }; TempDir *UpdateCorpusDatabaseTest::temp_dir_ = nullptr; -absl::NoDestructor UpdateCorpusDatabaseTest::centipede_std_out_{}; -absl::NoDestructor UpdateCorpusDatabaseTest::centipede_std_err_{}; +absl::NoDestructor + UpdateCorpusDatabaseTest::update_corpus_database_std_out_{}; +absl::NoDestructor + UpdateCorpusDatabaseTest::update_corpus_database_std_err_{}; -TEST_F(UpdateCorpusDatabaseTest, RunsFuzzTests) { - EXPECT_THAT(GetCentipedeStdErr(), +TEST_P(UpdateCorpusDatabaseTest, RunsFuzzTests) { + EXPECT_THAT(GetUpdateCorpusDatabaseStdErr(), AllOf(HasSubstr("Fuzzing FuzzTest.FailsInTwoWays"), HasSubstr("Fuzzing FuzzTest.FailsWithStackOverflow"))); } -TEST_F(UpdateCorpusDatabaseTest, UsesMultipleShardsForFuzzingAndDistillation) { +TEST_P(UpdateCorpusDatabaseTest, UsesMultipleShardsForFuzzingAndDistillation) { EXPECT_THAT( - GetCentipedeStdErr(), + GetUpdateCorpusDatabaseStdErr(), AllOf(HasSubstr("[S0.0] begin-fuzz"), HasSubstr("[S1.0] begin-fuzz"), HasSubstr("DISTILL[S.0]: Distilling to output shard 0"), HasSubstr("DISTILL[S.1]: Distilling to output shard 1"))); } -TEST_F(UpdateCorpusDatabaseTest, FindsAllCrashes) { +TEST_P(UpdateCorpusDatabaseTest, FindsAllCrashes) { EXPECT_THAT( - GetCentipedeStdErr(), + GetUpdateCorpusDatabaseStdErr(), AllOf(ContainsRegex(R"re(Failure\s*: GoogleTest assertion failure)re"), ContainsRegex(R"re(Failure\s*: heap-buffer-overflow)re"), ContainsRegex(R"re(Failure\s*: stack-limit-exceeded)re"))); } -TEST_F(UpdateCorpusDatabaseTest, ResumedFuzzTestRunsForRemainingTime) { +TEST_P(UpdateCorpusDatabaseTest, ResumedFuzzTestRunsForRemainingTime) { TempDir corpus_database; // 1st run that gets interrupted. - auto [fst_status, fst_std_out, fst_std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - corpus_database.dirname()), - CreateFuzzTestFlag("fuzz_for", "300s")}, - /*separator=*/" ")}, - // Disable symbolization to more quickly get to fuzzing. - {"symbolizer_path", ""}}, - // Stop the binary with SIGTERM before the fuzzing is done. + auto [fst_status, fst_std_out, fst_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "30s"}, + }, .timeout = absl::Seconds(10)}); - ASSERT_THAT(fst_status, Eq(Signal(SIGTERM))); // Adjust the fuzzing time so that only 1s remains. const absl::StatusOr fuzzing_time_file = FindFile(corpus_database.dirname(), "fuzzing_time"); - ASSERT_TRUE(fuzzing_time_file.ok()); + ASSERT_TRUE(fuzzing_time_file.ok()) << fst_std_err; ASSERT_TRUE(WriteFile(*fuzzing_time_file, "299s")); // 2nd run that resumes the fuzzing. - auto [snd_status, snd_std_out, snd_std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - corpus_database.dirname()), - CreateFuzzTestFlag("fuzz_for", "300s")}, - /*separator=*/" ")}, - // Disable symbolization to more quickly get to fuzzing. - {"symbolizer_path", ""}}, + auto [snd_status, snd_std_out, snd_std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = + { + {"corpus_database", corpus_database.dirname()}, + {"fuzz_for", "300s"}, + }, .timeout = absl::Seconds(10)}); EXPECT_THAT( @@ -172,16 +210,12 @@ TEST_F(UpdateCorpusDatabaseTest, ResumedFuzzTestRunsForRemainingTime) { HasSubstr("Fuzzing FuzzTest.FailsInTwoWays for 1s"))); } -TEST_F(UpdateCorpusDatabaseTest, ReplaysFuzzTestsInParallel) { - auto [status, std_out, std_err] = RunBinary( - CentipedePath(), - {.flags = {{"binary", - absl::StrJoin({GetCorpusDatabaseTestingBinaryPath(), - CreateFuzzTestFlag("corpus_database", - GetCorpusDatabasePath()), - CreateFuzzTestFlag("replay_corpus_for", "inf"), - CreateFuzzTestFlag("jobs", "2")}, - /*separator=*/" ")}}, +TEST_P(UpdateCorpusDatabaseTest, ReplaysFuzzTestsInParallel) { + auto [status, std_out, std_err] = RunBinaryMaybeWithCentipede( + GetCorpusDatabaseTestingBinaryPath(), + {.fuzztest_flags = {{"corpus_database", GetCorpusDatabasePath()}, + {"replay_corpus_for", "inf"}, + {"jobs", "2"}}, .timeout = absl::Seconds(30)}); EXPECT_THAT( @@ -191,5 +225,10 @@ TEST_F(UpdateCorpusDatabaseTest, ReplaysFuzzTestsInParallel) { HasSubstr("[S0.0] begin-fuzz"), HasSubstr("[S1.0] begin-fuzz"))); } +INSTANTIATE_TEST_SUITE_P( + UpdateCorpusDatabaseTestWithExecutionModel, UpdateCorpusDatabaseTest, + testing::ValuesIn({ExecutionModelParam::kSingleBinary, + ExecutionModelParam::kWithCentipedeBinary})); + } // namespace } // namespace fuzztest::internal diff --git a/e2e_tests/functional_test.cc b/e2e_tests/functional_test.cc index a5ccbbf4..7182ccad 100644 --- a/e2e_tests/functional_test.cc +++ b/e2e_tests/functional_test.cc @@ -20,6 +20,7 @@ #include // NOLINT #include #include +#include #include #include #include @@ -75,6 +76,17 @@ absl::flat_hash_map WithTestSanitizerOptions( return env; } +void ExpectTargetAbort(TerminationStatus status, absl::string_view std_err) { +#ifdef FUZZTEST_USE_CENTIPEDE + EXPECT_THAT(status, Ne(ExitCode(0))); + EXPECT_TRUE( + RE2::PartialMatch(std_err, absl::StrCat("Exit code\\s*:\\s*", SIGABRT))) + << std_err; +#else + EXPECT_THAT(status, Eq(Signal(SIGABRT))); +#endif +} + int CountSubstrs(absl::string_view haystack, absl::string_view needle) { int count = 0; while (true) { @@ -97,10 +109,12 @@ class UnitTestModeTest : public ::testing::Test { absl::string_view test_filter, absl::string_view target_binary = kDefaultTargetBinary, const absl::flat_hash_map& env = {}, - const absl::flat_hash_map& fuzzer_flags = {}) { + absl::flat_hash_map fuzzer_flags = {}) { + fuzzer_flags["print_subprocess_log"] = "true"; return RunBinary( BinaryPath(target_binary), - {.flags = {{GTEST_FLAG_PREFIX_ "filter", std::string(test_filter)}}, + {.flags = {{GTEST_FLAG_PREFIX_ "filter", std::string(test_filter)}, + {"symbolize_stacktrace", "0"}}, .fuzztest_flags = fuzzer_flags, .env = WithTestSanitizerOptions(env)}); } @@ -257,7 +271,7 @@ TEST_F(UnitTestModeTest, GoogleTestStaticTestSuiteFunctionsCalledInBalance) { TEST_F(UnitTestModeTest, GoogleTestWorksWithProtoExtensionsUsedInSeeds) { auto [status, std_out, std_err] = Run("MySuite.CheckProtoExtensions"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("Uses proto extensions")); } @@ -274,7 +288,7 @@ TEST_F(UnitTestModeTest, RepeatedFieldsHaveMinSizeWhenInitialized) { TEST_F(UnitTestModeTest, OptionalProtoFieldCanHaveNoValue) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenFieldI32HasNoValue"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, OptionalProtoFieldThatIsUnsetNeverHasValue) { @@ -318,26 +332,26 @@ TEST_F(UnitTestModeTest, TEST_F(UnitTestModeTest, RequiredProtoFieldThatIsNotAlwaysSetCanHaveNoValue) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenRequiredEnumFieldHasNoValue"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("cannot have null values")); } TEST_F(UnitTestModeTest, OptionalProtoFieldThatIsNotAlwaysSetCanHaveNoValue) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenOptionalFieldU32HasNoValue"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, ProtobufOfMutatesTheProto) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenI32IsSet"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("The field i32 is set!")); } TEST_F(UnitTestModeTest, ProtobufEnumEqualsLabel4) { auto [status, std_out, std_err] = Run("MySuite.FailsIfProtobufEnumEqualsLabel4"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT( std_err, HasSubstr("argument 0: fuzztest::internal::TestProtobuf::Label4")); @@ -345,7 +359,7 @@ TEST_F(UnitTestModeTest, ProtobufEnumEqualsLabel4) { TEST_F(UnitTestModeTest, WorksWithRecursiveStructs) { auto [status, std_out, std_err] = Run("MySuite.WorksWithRecursiveStructs"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); // Nullptr has multiple possible human-readable representations. EXPECT_THAT(std_err, AnyOf(HasSubstr("argument 0: LinkedList{0, 1}"), HasSubstr("argument 0: LinkedList{(nil), 1}"))); @@ -354,46 +368,46 @@ TEST_F(UnitTestModeTest, WorksWithRecursiveStructs) { TEST_F(UnitTestModeTest, WorksWithStructsWithConstructors) { auto [status, std_out, std_err] = Run("MySuite.WorksWithStructsWithConstructors"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: HasConstructor{1, \"abc\"}")); } TEST_F(UnitTestModeTest, WorksWithStructsWithEmptyTuples) { auto [status, std_out, std_err] = Run("MySuite.WorksWithStructsWithEmptyTuples"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: ContainsEmptyTuple{}")); } TEST_F(UnitTestModeTest, WorksWithEmptyStructs) { auto [status, std_out, std_err] = Run("MySuite.WorksWithEmptyStructs"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: Empty{}")); } TEST_F(UnitTestModeTest, WorksWithStructsWithEmptyFields) { auto [status, std_out, std_err] = Run("MySuite.WorksWithStructsWithEmptyFields"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: ContainsEmpty{Empty{}}")); } TEST_F(UnitTestModeTest, WorksWithEmptyInheritance) { auto [status, std_out, std_err] = Run("MySuite.WorksWithEmptyInheritance"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0: Child{0, \"abc\"}")); } TEST_F(UnitTestModeTest, ArbitraryWorksWithEmptyInheritance) { auto [status, std_out, std_err] = Run("MySuite.ArbitraryWorksWithEmptyInheritance"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("argument 0:")); } TEST_F(UnitTestModeTest, FlatMapCorrectlyPrintsValues) { auto [status, std_out, std_err] = Run("MySuite.FlatMapCorrectlyPrintsValues"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); // This is the argument to the output domain. EXPECT_THAT(std_err, HasSubstr("argument 0: {\"AAA\", \"BBB\"}")); // This is the argument to the input domain. @@ -402,24 +416,24 @@ TEST_F(UnitTestModeTest, FlatMapCorrectlyPrintsValues) { TEST_F(UnitTestModeTest, PrintsVeryLongInputsTrimmed) { auto [status, std_out, std_err] = Run("MySuite.LongInput"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("65 ...")); EXPECT_THAT(std_err, HasSubstr("A ...")); } TEST_F(UnitTestModeTest, PropertyFunctionAcceptsTupleOfItsSingleParameter) { auto [status, std_out, std_err] = Run("MySuite.UnpacksTupleOfOne"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, PropertyFunctionAcceptsTupleOfItsThreeParameters) { auto [status, std_out, std_err] = Run("MySuite.UnpacksTupleOfThree"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, PropertyFunctionAcceptsTupleContainingTuple) { auto [status, std_out, std_err] = Run("MySuite.UnpacksTupleContainingTuple"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, ProtoFieldsCanBeAlwaysSet) { @@ -465,7 +479,7 @@ TEST_F( TEST_F(UnitTestModeTest, DetectsRecursiveStructureIfOptionalsSetByDefault) { auto [status, std_out, std_err] = Run("MySuite.FailsIfCantInitializeProto"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("recursive fields")); } @@ -494,7 +508,7 @@ TEST_F(UnitTestModeTest, TEST_F(UnitTestModeTest, FailsWhenRepeatedFieldsSizeRangeIsInvalid) { auto [status, std_out, std_err] = Run("MySuite.FailsToInitializeIfRepeatedFieldsSizeRangeIsInvalid"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("size range is not valid")); } @@ -507,7 +521,7 @@ TEST_F(UnitTestModeTest, UsesPolicyProvidedDefaultDomainForProtos) { TEST_F(UnitTestModeTest, ChecksTypeOfProvidedDefaultDomainForProtos) { auto [status, std_out, std_err] = Run("MySuite.FailsWhenWrongDefaultProtobufDomainIsProvided"); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); EXPECT_THAT(std_err, HasSubstr("does not match the expected message type")); } @@ -551,7 +565,7 @@ TEST_F(UnitTestModeTest, StackLimitWorks) { /*env=*/{}, /*fuzzer_flags=*/{{"stack_limit_kb", "1000"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); ExpectStackLimitExceededMessage(std_err, 1024000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, RssLimitFlagWorks) { @@ -560,7 +574,7 @@ TEST_F(UnitTestModeTest, RssLimitFlagWorks) { /*env=*/{}, /*fuzzer_flags=*/{{"rss_limit_mb", "1024"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex(absl::StrCat("RSS limit exceeded"))); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, TimeLimitFlagWorks) { @@ -570,7 +584,7 @@ TEST_F(UnitTestModeTest, TimeLimitFlagWorks) { /*fuzzer_flags=*/{{"time_limit_per_input", "1s"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex("Per-input timeout exceeded")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(UnitTestModeTest, TestIsSkippedWhenRequestedInFixturePerTest) { @@ -647,12 +661,13 @@ TEST_F(GetRandomValueTest, SettingPrngSeedReproducesValue) { class GenericCommandLineInterfaceTest : public ::testing::Test { protected: RunResults RunWith( - const absl::flat_hash_map& flags, + absl::flat_hash_map flags, const absl::flat_hash_map& env = {}, absl::Duration timeout = absl::Minutes(10), absl::string_view binary = kDefaultTargetBinary, const absl::flat_hash_map& non_fuzztest_flags = {}) { + flags["print_subprocess_log"] = "true"; return RunBinary(BinaryPath(binary), RunOptions{.flags = non_fuzztest_flags, .fuzztest_flags = flags, @@ -723,14 +738,14 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, TEST_F(FuzzingModeCommandLineInterfaceTest, RunsAbortTestAndDetectsAbort) { auto [status, std_out, std_err] = RunWith({{"fuzz", "MySuite.Aborts"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, FuzzTestCanBeSelectedForFuzzingUsingSubstring) { auto [status, std_out, std_err] = RunWith({{"fuzz", "Abort"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, @@ -769,7 +784,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, ReproducerIsDumpedWhenEnvVarIsSet) { RunWith({{"fuzz", "MySuite.StringFast"}}, {{"FUZZTEST_REPRODUCERS_OUT_DIR", out_dir.dirname()}}); EXPECT_THAT(std_err, HasSubstr("argument 0: \"Fuzz")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); auto replay_files = ReadFileOrDirectory(out_dir.dirname()); ASSERT_EQ(replay_files.size(), 1) << std_err; @@ -1034,6 +1049,10 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, MinimizerFindsSmallerInput) { TEST_F(FuzzingModeCommandLineInterfaceTest, FuzzerStatsArePrintedOnTermination) { +#ifdef FUZZTEST_USE_CENTIPEDE + GTEST_SKIP() << "Skip fuzzer stats test when running with Centipede because " + "stats are not printed."; +#endif auto [status, std_out, std_err] = RunWith({{"fuzz", "MySuite.PassesWithPositiveInput"}}, /*env=*/{}, @@ -1051,7 +1070,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, SilenceTargetWorking) { EXPECT_THAT(std_out, Not(HasSubstr("Hello World from target stdout"))); EXPECT_THAT(std_err, HasSubstr("=== Fuzzing stats")); EXPECT_THAT(std_err, Not(HasSubstr("Hello World from target stderr"))); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, NonFatalFailureAllowsMinimization) { @@ -1066,7 +1085,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, NonFatalFailureAllowsMinimization) { // "larger" inputs also trigger the failure. EXPECT_THAT(std_err, HasSubstr("argument 0: \"0123\"")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, GoogleTestHasCurrentTestInfo) { @@ -1083,7 +1102,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, ConfiguresStackLimitByFlag) { {"stack_limit_kb", "1000"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); ExpectStackLimitExceededMessage(std_err, 1024000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, @@ -1098,7 +1117,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, "is going to be deprecated soon. Consider switching to ", CreateFuzzTestFlag("stack_limit_kb", ""), " flag."))); ExpectStackLimitExceededMessage(std_err, 512000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, @@ -1109,15 +1128,15 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, {{"FUZZTEST_STACK_LIMIT", "512000"}}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); ExpectStackLimitExceededMessage(std_err, 512000); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, DoesNotPrintWarningForDisabledLimitFlagsByDefault) { - auto [status, std_out, std_err] = - RunWith({{"fuzz", "MySuite.PassesWithPositiveInput"}}, - /*env=*/{}, - /*timeout=*/absl::Seconds(10)); + auto [status, std_out, std_err] = RunWith( + {{"fuzz", "MySuite.PassesWithPositiveInput"}, {"fuzz_for", "10s"}}, + /*env=*/{}, + /*timeout=*/absl::Seconds(20)); EXPECT_THAT(std_err, Not(HasSubstr("limit is specified but will be ignored"))); EXPECT_THAT(status, Eq(ExitCode(0))); @@ -1129,7 +1148,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, RssLimitFlagWorks) { /*env=*/{}, /*timeout=*/absl::Seconds(10)); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex(absl::StrCat("RSS limit exceeded"))); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } TEST_F(FuzzingModeCommandLineInterfaceTest, TimeLimitFlagWorks) { @@ -1138,7 +1157,7 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, TimeLimitFlagWorks) { /*env=*/{}); EXPECT_THAT(std_err, HasSubstr("argument 0: ")); EXPECT_THAT(std_err, ContainsRegex("Per-input timeout exceeded")); - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + ExpectTargetAbort(status, std_err); } // TODO: b/340232436 - Once fixed, remove this test since we will no longer need @@ -1204,19 +1223,22 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, UsesCentipedeBinaryWhenEnvIsSet) { /*timeout=*/absl::Minutes(1), "testdata/unit_test_and_fuzz_tests"); EXPECT_THAT( std_err, - HasSubstr("Starting the update of the corpus database for fuzz tests")); + HasSubstr("Starting the update of the corpus database for fuzz tests")) + << std_err; EXPECT_THAT(std_err, HasSubstr("FuzzTest.AlwaysPasses")); EXPECT_THAT(status, Eq(ExitCode(0))); } -struct ExecutionModelParam { - bool multi_process; +enum class ExecutionModelParam { + kSingleBinary, + kWithCentipedeBinary, }; std::vector GetAvailableExecutionModels() { - std::vector results = {{/*multi_process=*/false}}; + std::vector results = { + ExecutionModelParam::kSingleBinary}; #ifdef FUZZTEST_USE_CENTIPEDE - results.push_back({/*multi_process=*/true}); + results.push_back(ExecutionModelParam::kWithCentipedeBinary); #endif return results; } @@ -1242,25 +1264,30 @@ class FuzzingModeFixtureTest } RunResults Run(absl::string_view test_name, int iterations) { - if (GetParam().multi_process) { - TempDir workdir; - return RunBinary( - CentipedePath(), - {.flags = {{"print_runner_log", "true"}, - {"exit_on_crash", "true"}, - {"workdir", workdir.dirname()}, - {"binary", - absl::StrCat(BinaryPath(kDefaultTargetBinary), " ", - CreateFuzzTestFlag("fuzz", test_name))}, - {"num_runs", absl::StrCat(iterations)}}, - .timeout = absl::InfiniteDuration()}); - } else { - return RunBinary( - BinaryPath(kDefaultTargetBinary), - {.fuzztest_flags = {{"fuzz", std::string(test_name)}}, - .env = {{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}}, - .timeout = absl::InfiniteDuration()}); + switch (GetParam()) { + case ExecutionModelParam::kSingleBinary: + return RunBinary( + BinaryPath(kDefaultTargetBinary), + {.fuzztest_flags = {{"fuzz", std::string(test_name)}, + {"print_subprocess_log", "true"}}, + .env = {{"FUZZTEST_MAX_FUZZING_RUNS", absl::StrCat(iterations)}}, + .timeout = absl::InfiniteDuration()}); + case ExecutionModelParam::kWithCentipedeBinary: { + TempDir workdir; + return RunBinary( + CentipedePath(), + {.flags = {{"print_runner_log", "true"}, + {"exit_on_crash", "true"}, + {"workdir", workdir.dirname()}, + {"binary", + absl::StrCat(BinaryPath(kDefaultTargetBinary), " ", + CreateFuzzTestFlag("fuzz", test_name))}, + {"num_runs", absl::StrCat(iterations)}}, + .timeout = absl::InfiniteDuration()}); + } } + fprintf(stderr, "Unsupported execution model!\n"); + std::abort(); } }; @@ -1384,7 +1411,7 @@ class FuzzingModeCrashFindingTest // There are however env vars that we do want to propagate, which // we now need to do explicitly. env = WithTestSanitizerOptions(std::move(env)); - if (GetParam().multi_process) { + if (GetParam() == ExecutionModelParam::kWithCentipedeBinary) { TempDir workdir; return RunBinary( CentipedePath(), @@ -1399,20 +1426,10 @@ class FuzzingModeCrashFindingTest .timeout = timeout + absl::Seconds(10)}); } else { return RunBinary(BinaryPath(target_binary), - {.fuzztest_flags = {{"fuzz", std::string(test_name)}}, + {.fuzztest_flags = {{"fuzz", std::string(test_name)}, + {"fuzz_for", absl::StrCat(timeout)}}, .env = std::move(env), - .timeout = timeout}); - } - } - - void ExpectTargetAbort(TerminationStatus status, absl::string_view std_err) { - if (GetParam().multi_process) { - EXPECT_THAT(status, Ne(ExitCode(0))); - EXPECT_TRUE(RE2::PartialMatch( - std_err, absl::StrCat("Exit code\\s*:\\s*", SIGABRT))) - << std_err; - } else { - EXPECT_THAT(status, Eq(Signal(SIGABRT))); + .timeout = timeout + absl::Seconds(10)}); } } }; @@ -1802,7 +1819,10 @@ TEST_P(FuzzingModeCrashFindingTest, TempDir out_dir; auto [status, std_out, std_err] = Run("LLVMFuzzer.TestOneInput", "testdata/llvm_fuzzer_with_custom_mutator", - /*env=*/{}, /*timeout=*/absl::Seconds(30)); + /*env=*/ + { + }, + /*timeout=*/absl::Seconds(30)); EXPECT_THAT(std_err, HasSubstr("argument 0: \"ahmfn\"")); ExpectTargetAbort(status, std_err); } diff --git a/fuzztest/BUILD b/fuzztest/BUILD index 4dcdb3ce..5f462a25 100644 --- a/fuzztest/BUILD +++ b/fuzztest/BUILD @@ -213,9 +213,9 @@ cc_library( deps = [ ":any", ":configuration", - ":corpus_database", ":domain_core", ":fixture_driver", + ":flag_name", ":logging", ":runtime", ":table_of_recent_compares", @@ -224,12 +224,14 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/random", "@com_google_absl//absl/random:distributions", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:string_view", "@com_google_absl//absl/time", "@com_google_absl//absl/types:span", "@com_google_fuzztest//centipede:centipede_callbacks", + "@com_google_fuzztest//centipede:centipede_default_callbacks", "@com_google_fuzztest//centipede:centipede_interface", "@com_google_fuzztest//centipede:centipede_runner_no_main", "@com_google_fuzztest//centipede:environment", diff --git a/fuzztest/init_fuzztest.cc b/fuzztest/init_fuzztest.cc index cecaf31b..4d847962 100644 --- a/fuzztest/init_fuzztest.cc +++ b/fuzztest/init_fuzztest.cc @@ -163,6 +163,38 @@ FUZZTEST_DEFINE_FLAG(std::optional, jobs, std::nullopt, "The number of fuzzing jobs to run in parallel. If " "unspecified, the number of jobs is 1."); +FUZZTEST_DEFINE_FLAG( + std::optional, internal_override_fuzz_test, std::nullopt, + "Internal-only flag - do not use directly. If set, only perform operations " + "for the exact fuzz test regardless of other flags.") + .OnUpdate([] { + FUZZTEST_INTERNAL_CHECK_PRECONDITION( + !absl::GetFlag(FUZZTEST_FLAG(internal_override_fuzz_test)) + .has_value() || + getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr, + "must not set --" FUZZTEST_FLAG_PREFIX + "internal_override_fuzz_test directly"); + }); + +FUZZTEST_DEFINE_FLAG( + absl::Duration, internal_override_total_time_limit, + absl::InfiniteDuration(), + "Internal-only flag - do not use directly. If --" FUZZTEST_FLAG_PREFIX + "internal_override_fuzz_test is set, use this total time limit regardless " + "of other flags.") + .OnUpdate([] { + FUZZTEST_INTERNAL_CHECK_PRECONDITION( + absl::GetFlag(FUZZTEST_FLAG(internal_override_total_time_limit)) == + absl::InfiniteDuration() || + getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr, + "must not set --" FUZZTEST_FLAG_PREFIX + "internal_override_total_time_limit directly"); + }); + +FUZZTEST_DEFINE_FLAG( + bool, print_subprocess_log, false, + "If set, print the log of the subprocesses spawned by FuzzTest."); + namespace fuzztest { std::vector ListRegisteredTests() { @@ -239,16 +271,24 @@ std::optional GetReplayCorpusTime() { internal::Configuration CreateConfigurationsFromFlags( absl::string_view binary_identifier) { - bool reproduce_findings_as_separate_tests = + const bool reproduce_findings_as_separate_tests = absl::GetFlag(FUZZTEST_FLAG(reproduce_findings_as_separate_tests)); - std::optional fuzzing_time_limit = GetFuzzingTime(); - std::optional replay_corpus_time_limit = + const std::optional fuzzing_time_limit = GetFuzzingTime(); + const std::optional replay_corpus_time_limit = GetReplayCorpusTime(); - absl::Duration time_limit = fuzzing_time_limit ? *fuzzing_time_limit - : replay_corpus_time_limit - ? *replay_corpus_time_limit - : absl::ZeroDuration(); - std::optional jobs = absl::GetFlag(FUZZTEST_FLAG(jobs)); + const auto override_fuzz_test = + absl::GetFlag(FUZZTEST_FLAG(internal_override_fuzz_test)); + const absl::Duration time_limit = + override_fuzz_test.has_value() + ? absl::GetFlag(FUZZTEST_FLAG(internal_override_total_time_limit)) + : fuzzing_time_limit ? *fuzzing_time_limit + : replay_corpus_time_limit ? *replay_corpus_time_limit + : absl::ZeroDuration(); + const auto time_budget_type = + override_fuzz_test.has_value() + ? internal::TimeBudgetType::kTotal + : absl::GetFlag(FUZZTEST_FLAG(time_budget_type)); + const std::optional jobs = absl::GetFlag(FUZZTEST_FLAG(jobs)); FUZZTEST_INTERNAL_CHECK(!jobs.has_value() || *jobs > 0, "If specified, --", FUZZTEST_FLAG(jobs).Name(), " must be positive."); return internal::Configuration{ @@ -258,12 +298,15 @@ internal::Configuration CreateConfigurationsFromFlags( /*fuzz_tests=*/ListRegisteredTests(), /*fuzz_tests_in_current_shard=*/ListRegisteredTests(), reproduce_findings_as_separate_tests, - /*only_replay_corpus=*/ + /*replay_corpus=*/ + fuzzing_time_limit.has_value() || replay_corpus_time_limit.has_value(), + /*only_replay=*/ replay_corpus_time_limit.has_value(), + absl::GetFlag(FUZZTEST_FLAG(print_subprocess_log)), /*stack_limit=*/absl::GetFlag(FUZZTEST_FLAG(stack_limit_kb)) * 1024, /*rss_limit=*/absl::GetFlag(FUZZTEST_FLAG(rss_limit_mb)) * 1024 * 1024, absl::GetFlag(FUZZTEST_FLAG(time_limit_per_input)), time_limit, - absl::GetFlag(FUZZTEST_FLAG(time_budget_type)), jobs.value_or(0)}; + time_budget_type, jobs.value_or(0)}; } } // namespace @@ -297,14 +340,30 @@ void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { const auto test_to_fuzz = absl::GetFlag(FUZZTEST_FLAG(fuzz)); const auto test_to_replay_corpus = absl::GetFlag(FUZZTEST_FLAG(replay_corpus)); - const auto specified_test = - test_to_fuzz != kUnspecified ? test_to_fuzz : test_to_replay_corpus; - const bool is_test_specified = specified_test != kUnspecified; - if (is_test_specified) { + const auto specified_test = []() -> std::optional { + if (auto internal_selected_test = + absl::GetFlag(FUZZTEST_FLAG(internal_override_fuzz_test)); + internal_selected_test.has_value()) { + return internal_selected_test; + } + if (auto test_to_fuzz = absl::GetFlag(FUZZTEST_FLAG(fuzz)); + test_to_fuzz != kUnspecified) { + return test_to_fuzz; + } + if (auto test_to_replay_corpus = + absl::GetFlag(FUZZTEST_FLAG(replay_corpus)); + test_to_replay_corpus != kUnspecified) + return test_to_replay_corpus; + return std::nullopt; + }(); + if (specified_test.has_value()) { const std::string matching_fuzz_test = - GetMatchingFuzzTestOrExit(specified_test); + GetMatchingFuzzTestOrExit(*specified_test); // Delegate the test to GoogleTest. GTEST_FLAG_SET(filter, matching_fuzz_test); + // Do not list tests when the fuzz test is specified. Needed for + // multi-process mode to work where args are passed through. + GTEST_FLAG_SET(list_tests, false); } std::string derived_binary_id = @@ -318,7 +377,7 @@ void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { const bool is_fuzzing_or_replaying = (fuzzing_time_limit || replay_corpus_time_limit); - if (is_fuzzing_or_replaying && !is_test_specified) { + if (is_fuzzing_or_replaying && !specified_test.has_value()) { absl::flat_hash_set fuzz_tests = { configuration.fuzz_tests.begin(), configuration.fuzz_tests.end()}; std::vector non_fuzz_tests; @@ -343,11 +402,8 @@ void InitFuzzTest(int* argc, char*** argv, std::string_view binary_id) { GTEST_FLAG_SET(filter, filter); } } - const bool is_runner_mode = std::getenv("CENTIPEDE_RUNNER_FLAGS") != nullptr; - const bool is_fuzzing_mode = (is_runner_mode && is_fuzzing_or_replaying) || - fuzzing_time_limit.has_value(); const RunMode run_mode = - is_fuzzing_mode ? RunMode::kFuzz : RunMode::kUnitTest; + fuzzing_time_limit.has_value() ? RunMode::kFuzz : RunMode::kUnitTest; // TODO(b/307513669): Use the Configuration class instead of Runtime. internal::Runtime::instance().SetRunMode(run_mode); } diff --git a/fuzztest/internal/centipede_adaptor.cc b/fuzztest/internal/centipede_adaptor.cc index 26a3cb12..95d8fd8e 100644 --- a/fuzztest/internal/centipede_adaptor.cc +++ b/fuzztest/internal/centipede_adaptor.cc @@ -15,6 +15,13 @@ #include "./fuzztest/internal/centipede_adaptor.h" #include +#ifdef __APPLE__ +#include +#else // __APPLE__ +#include // ARG_MAX +#endif // __APPLE__ +#include +#include #include #include @@ -41,15 +48,18 @@ #include "absl/memory/memory.h" #include "absl/random/distributions.h" #include "absl/random/random.h" +#include "absl/status/status.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "absl/time/clock.h" #include "absl/time/time.h" #include "absl/types/span.h" #include "./centipede/centipede_callbacks.h" +#include "./centipede/centipede_default_callbacks.h" #include "./centipede/centipede_interface.h" #include "./centipede/environment.h" #include "./centipede/mutation_input.h" @@ -61,9 +71,9 @@ #include "./common/defs.h" #include "./fuzztest/internal/any.h" #include "./fuzztest/internal/configuration.h" -#include "./fuzztest/internal/corpus_database.h" #include "./fuzztest/internal/domains/domain.h" #include "./fuzztest/internal/fixture_driver.h" +#include "./fuzztest/internal/flag_name.h" #include "./fuzztest/internal/logging.h" #include "./fuzztest/internal/runtime.h" #include "./fuzztest/internal/table_of_recent_compares.h" @@ -97,6 +107,64 @@ class TempDir { std::string path_; }; +absl::StatusOr> GetProcessArgs() { + std::vector results; +#ifdef __APPLE__ + char args[ARG_MAX]; + size_t args_size = sizeof(args); + int mib[] = {CTL_KERN, KERN_PROCARGS2, getpid()}; + int rv = + sysctl(mib, sizeof(mib) / sizeof(mib[0]), args, &args_size, nullptr, 0); + if (rv != 0) { + return absl::InternalError( + "GetEnv: sysctl({CTK_KERN, KERN_PROCARGS2, ...}) failed"); + } + if (args_size < sizeof(int)) { + return absl::InternalError("GetEnv: args_size too small"); + } + int argc = 0; + memcpy(&argc, &args[0], sizeof(argc)); + size_t start_pos = sizeof(argc); + // Find the end of the executable path. + while (start_pos < args_size && args[start_pos] != 0) ++start_pos; + if (start_pos == args_size) { + return absl::NotFoundError("nothing after executable path"); + } + // Find the beginning of the string area. + while (start_pos < args_size && args[start_pos] == 0) ++start_pos; + if (start_pos == args_size) { + return absl::NotFoundError("nothing after executable path"); + } + // Get the first argc c-strings without exceeding the boundary. + for (int i = 0; i < argc; ++i) { + const size_t current_argv_pos = start_pos; + while (start_pos < args_size && args[start_pos] != 0) ++start_pos; + results.emplace_back(&args[current_argv_pos], start_pos - current_argv_pos); + ++start_pos; + } +#else + const int fd = open("/proc/self/cmdline", O_RDONLY); + if (fd < 0) return absl::InternalError("cannot open /proc/self/cmdline\n"); + char args[4096]; + const ssize_t args_size = read(fd, args, sizeof(args)); + if (args_size < 0) + return absl::InternalError("cannot open /proc/self/cmdline\n"); + if (close(fd) != 0) LOG(ERROR) << "cannot close /proc/self/cmdline\n"; + size_t start_pos = 0; + while (start_pos < args_size) { + const size_t current_argv_pos = start_pos; + while (start_pos < args_size && args[start_pos] != 0) ++start_pos; + results.emplace_back(&args[current_argv_pos], start_pos - current_argv_pos); + ++start_pos; + } +#endif + return results; +} + +std::string ShellEscape(absl::string_view str) { + return absl::StrCat("'", absl::StrReplaceAll(str, {{"'", "'\\''"}}), "'"); +} + // TODO(xinhaoyuan): Consider passing rng seeds from the engine. std::seed_seq GetRandomSeed() { const size_t seed = time(nullptr) + getpid() + @@ -119,33 +187,88 @@ centipede::Environment CreateDefaultCentipedeEnvironment() { centipede::Environment CreateCentipedeEnvironmentFromFuzzTestFlags( const Configuration& configuration, absl::string_view workdir, - absl::string_view test_name) { + absl::string_view test_name, RunMode run_mode) { centipede::Environment env = CreateDefaultCentipedeEnvironment(); + constexpr absl::Duration kUnitTestDefaultDuration = absl::Seconds(3); + env.fuzztest_single_test_mode = true; + env.populate_binary_info = false; + const auto args = GetProcessArgs(); + FUZZTEST_INTERNAL_CHECK( + args.ok(), + absl::StrCat("failed to get the original process args: ", args.status())); + env.binary.clear(); + for (const auto& arg : *args) { + // We need shell escaping, because parts of binary_arg will be passed to + // system(), which uses the default shell. + absl::StrAppend(&env.binary, env.binary.empty() ? "" : " ", + ShellEscape(arg)); + } + absl::StrAppend( + &env.binary, + " --" FUZZTEST_FLAG_PREFIX "internal_override_fuzz_test=", test_name); + absl::Duration total_time_limit = configuration.GetTimeLimitPerTest(); + if (total_time_limit == absl::ZeroDuration() && + run_mode == RunMode::kUnitTest) { + total_time_limit = kUnitTestDefaultDuration; + } + absl::StrAppend(&env.binary, + " --" FUZZTEST_FLAG_PREFIX + "internal_override_total_time_limit=", + total_time_limit); + env.coverage_binary = (*args)[0]; + env.exit_on_crash = run_mode == RunMode::kUnitTest || + configuration.corpus_database.empty() || + configuration.crashing_input_to_reproduce.has_value(); + env.print_runner_log = configuration.print_subprocess_log; env.workdir = workdir; - env.exit_on_crash = true; - // Populating the PC table in single-process mode is not implemented. - env.require_pc_table = false; - const auto time_limit_per_test = configuration.GetTimeLimitPerTest(); - if (time_limit_per_test != absl::InfiniteDuration()) { - absl::FPrintF(GetStderr(), "[.] Fuzzing timeout set to: %s\n", - absl::FormatDuration(time_limit_per_test)); - env.stop_at = absl::Now() + time_limit_per_test; - } - env.first_corpus_dir_output_only = true; - if (const char* corpus_out_dir_chars = getenv("FUZZTEST_TESTSUITE_OUT_DIR")) { - env.corpus_dir.push_back(corpus_out_dir_chars); + if (configuration.corpus_database.empty()) { + if (total_time_limit != absl::InfiniteDuration()) { + absl::FPrintF(GetStderr(), "[.] Fuzzing timeout set to: %s\n", + absl::FormatDuration(total_time_limit)); + env.stop_at = absl::Now() + total_time_limit; + } + env.first_corpus_dir_output_only = true; + if (const char* corpus_out_dir_chars = + getenv("FUZZTEST_TESTSUITE_OUT_DIR")) { + env.corpus_dir.push_back(corpus_out_dir_chars); + } else { + env.corpus_dir.push_back(""); + } + if (const char* corpus_in_dir_chars = getenv("FUZZTEST_TESTSUITE_IN_DIR")) { + env.corpus_dir.push_back(corpus_in_dir_chars); + } + if (const char* max_fuzzing_runs = getenv("FUZZTEST_MAX_FUZZING_RUNS")) { + if (!absl::SimpleAtoi(max_fuzzing_runs, &env.num_runs)) { + absl::FPrintF( + GetStderr(), + "[!] Cannot parse env FUZZTEST_MAX_FUZZING_RUNS=%s - will " + "not limit fuzzing runs.\n", + max_fuzzing_runs); + env.num_runs = std::numeric_limits::max(); + } + } } else { - env.corpus_dir.push_back(""); - } - if (const char* corpus_in_dir_chars = getenv("FUZZTEST_TESTSUITE_IN_DIR")) - env.corpus_dir.push_back(corpus_in_dir_chars); - if (const char* max_fuzzing_runs = getenv("FUZZTEST_MAX_FUZZING_RUNS")) { - if (!absl::SimpleAtoi(max_fuzzing_runs, &env.num_runs)) { + // Not setting env.stop_at since current update_corpus logic in Centipede + // would propagate that. + if (getenv("FUZZTEST_TESTSUITE_OUT_DIR")) { absl::FPrintF(GetStderr(), - "[!] Cannot parse env FUZZTEST_MAX_FUZZING_RUNS=%s - will " - "not limit fuzzing runs.\n", - max_fuzzing_runs); - env.num_runs = std::numeric_limits::max(); + "[!] Ignoring FUZZTEST_TESTSUITE_OUT_DIR when the corpus " + "database is set.\n"); + } + if (getenv("FUZZTEST_TESTSUITE_IN_DIR")) { + absl::FPrintF(GetStderr(), + "[!] Ignoring FUZZTEST_TESTSUITE_IN_DIR when the corpus " + "database is set.\n"); + } + if (getenv("FUZZTEST_MINIMIZE_TESTSUITE_DIR")) { + absl::FPrintF(GetStderr(), + "[!] Ignoring FUZZTEST_MINIMIZE_TESTSUITE_DIR when the " + "corpus database is set.\n"); + } + if (const char* max_fuzzing_runs = getenv("FUZZTEST_MAX_FUZZING_RUNS")) { + absl::FPrintF(GetStderr(), + "[!] Ignoring FUZZTEST_MAX_FUZZING_RUNS when the " + "corpus database is set.\n"); } } return env; @@ -165,6 +288,13 @@ class CentipedeAdaptorRunnerCallbacks : public centipede::RunnerCallbacks { prng_(GetRandomSeed()) {} bool Execute(centipede::ByteSpan input) override { + if (!domain_setup_is_checked_) { + // Create a new domain input to trigger any domain setup + // failures here. (e.g. Ineffective Filter) + fuzzer_impl_.params_domain_.Init(prng_); + domain_setup_is_checked_ = true; + } + auto parsed_input = fuzzer_impl_.TryParse({(char*)input.data(), input.size()}); if (parsed_input.ok()) { @@ -178,13 +308,6 @@ class CentipedeAdaptorRunnerCallbacks : public centipede::RunnerCallbacks { std::function seed_callback) override { std::vector seeds = fuzzer_impl_.fixture_driver_->GetSeeds(); - CorpusDatabase corpus_database(configuration_); - fuzzer_impl_.ForEachInput( - corpus_database.GetCoverageInputsIfAny(fuzzer_impl_.test_.full_name()), - [&](absl::string_view /*file_path*/, std::optional /*blob_idx*/, - FuzzTestFuzzerImpl::Input input) { - seeds.push_back(std::move(input.args)); - }); constexpr int kInitialValuesInSeeds = 32; for (int i = 0; i < kInitialValuesInSeeds; ++i) { seeds.push_back(fuzzer_impl_.params_domain_.Init(prng_)); @@ -229,7 +352,7 @@ class CentipedeAdaptorRunnerCallbacks : public centipede::RunnerCallbacks { size_t num_mutants, std::function new_mutant_callback) override { if (inputs.empty()) return false; - SetMetadata(inputs[0].metadata); + if (runtime_.run_mode() == RunMode::kFuzz) SetMetadata(inputs[0].metadata); for (size_t i = 0; i < num_mutants; ++i) { const auto choice = absl::Uniform(prng_, 0, 1); std::string mutant_data; @@ -300,130 +423,13 @@ class CentipedeAdaptorRunnerCallbacks : public centipede::RunnerCallbacks { Runtime& runtime_; FuzzTestFuzzerImpl& fuzzer_impl_; const Configuration& configuration_; + bool domain_setup_is_checked_ = false; std::unique_ptr cmp_tables_; absl::BitGen prng_; }; namespace { -class CentipedeAdaptorEngineCallbacks : public centipede::CentipedeCallbacks { - public: - CentipedeAdaptorEngineCallbacks(const centipede::Environment& env, - Runtime* runtime, - FuzzTestFuzzerImpl* fuzzer_impl, - const Configuration* configuration) - : centipede::CentipedeCallbacks(env), - runtime_(*runtime), - runner_callbacks_(runtime, fuzzer_impl, configuration), - batch_result_buffer_size_(env.shmem_size_mb * 1024 * 1024), - batch_result_buffer_(nullptr) {} - - ~CentipedeAdaptorEngineCallbacks() { - if (batch_result_buffer_ != nullptr) - munmap(batch_result_buffer_, batch_result_buffer_size_); - } - - bool Execute(std::string_view binary, - const std::vector& inputs, - centipede::BatchResult& batch_result) override { - // Execute the test in-process. - batch_result.ClearAndResize(inputs.size()); - size_t buffer_offset = 0; - if (batch_result_buffer_ == nullptr) { - // Use mmap which allocates memory on demand to reduce sanitizer overhead. - batch_result_buffer_ = static_cast( - mmap(nullptr, batch_result_buffer_size_, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); - FUZZTEST_INTERNAL_CHECK( - batch_result_buffer_ != MAP_FAILED, - "Cannot mmap anonymous memory for batch result buffer"); - } - CentipedeBeginExecutionBatch(); - for (const auto& input : inputs) { - if (runtime_.termination_requested()) break; - if (buffer_offset >= batch_result_buffer_size_) break; - runner_callbacks_.Execute(input); - buffer_offset += CentipedeGetExecutionResult( - batch_result_buffer_ + buffer_offset, - batch_result_buffer_size_ - buffer_offset); - } - CentipedeEndExecutionBatch(); - if (buffer_offset > 0) { - centipede::BlobSequence batch_result_blobseq(batch_result_buffer_, - buffer_offset); - batch_result.Read(batch_result_blobseq); - } - if (runtime_.termination_requested() && !centipede::ShouldStop()) { - absl::FPrintF(GetStderr(), "[.] Early termination requested.\n"); - centipede::RequestEarlyStop(0); - } - return true; - } - - size_t GetSeeds(size_t num_seeds, - std::vector& seeds) override { - seeds.clear(); - size_t num_avail_seeds = 0; - runner_callbacks_.GetSeeds([&](centipede::ByteSpan seed) { - ++num_avail_seeds; - if (seeds.size() < num_seeds) { - seeds.emplace_back(seed.begin(), seed.end()); - } - }); - return num_avail_seeds; - } - - std::string GetSerializedTargetConfig() override { - return runner_callbacks_.GetSerializedTargetConfig(); - } - - void Mutate(const std::vector& inputs, - size_t num_mutants, - std::vector& mutants) override { - mutants.clear(); - runner_callbacks_.Mutate( - inputs, num_mutants, [&](centipede::ByteSpan mutant) { - mutants.emplace_back(mutant.begin(), mutant.end()); - }); - if (runtime_.termination_requested() && !centipede::ShouldStop()) { - absl::FPrintF(GetStderr(), "[.] Early termination requested.\n"); - centipede::RequestEarlyStop(0); - } - } - - private: - Runtime& runtime_; - CentipedeAdaptorRunnerCallbacks runner_callbacks_; - size_t batch_result_buffer_size_; - uint8_t* batch_result_buffer_; -}; - -class CentipedeAdaptorEngineCallbacksFactory - : public centipede::CentipedeCallbacksFactory { - public: - CentipedeAdaptorEngineCallbacksFactory(Runtime* runtime, - FuzzTestFuzzerImpl* fuzzer_impl, - const Configuration* configuration) - : runtime_(runtime), - fuzzer_impl_(fuzzer_impl), - configuration_(configuration) {} - - centipede::CentipedeCallbacks* create( - const centipede::Environment& env) override { - return new CentipedeAdaptorEngineCallbacks(env, runtime_, fuzzer_impl_, - configuration_); - } - - void destroy(centipede::CentipedeCallbacks* callbacks) override { - delete callbacks; - } - - private: - Runtime* runtime_; - FuzzTestFuzzerImpl* fuzzer_impl_; - const Configuration* configuration_; -}; - void PopulateTestLimitsToCentipedeRunner(const Configuration& configuration) { if (const size_t stack_limit = GetStackLimitFromEnvOrConfiguration(configuration); @@ -518,32 +524,43 @@ CentipedeFuzzerAdaptor::CentipedeFuzzerAdaptor( "Invalid fixture driver!"); } -void CentipedeFuzzerAdaptor::RunInUnitTestMode( +bool CentipedeFuzzerAdaptor::RunInUnitTestMode( const Configuration& configuration) { - centipede_fixture_driver_->set_configuration(&configuration); - CentipedeBeginExecutionBatch(); - fuzzer_impl_.RunInUnitTestMode(configuration); - CentipedeEndExecutionBatch(); + return Run(/*argc=*/nullptr, /*argv=*/nullptr, RunMode::kUnitTest, + configuration); } -int CentipedeFuzzerAdaptor::RunInFuzzingMode( +bool CentipedeFuzzerAdaptor::RunInFuzzingMode( int* argc, char*** argv, const Configuration& configuration) { + return Run(argc, argv, RunMode::kFuzz, configuration); +} + +bool CentipedeFuzzerAdaptor::Run(int* argc, char*** argv, RunMode mode, + const Configuration& configuration) { centipede_fixture_driver_->set_configuration(&configuration); - runtime_.SetRunMode(RunMode::kFuzz); - runtime_.SetSkippingRequested(false); - runtime_.SetCurrentTest(&test_, &configuration); - if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr(); - runtime_.EnableReporter(&fuzzer_impl_.stats_, [] { return absl::Now(); }); - fuzzer_impl_.fixture_driver_->SetUpFuzzTest(); - // Always create a new domain input to trigger any domain setup - // failures here. (e.g. Ineffective Filter) - FuzzTestFuzzerImpl::PRNG prng; - fuzzer_impl_.params_domain_.Init(prng); - bool print_final_stats = true; // When the CENTIPEDE_RUNNER_FLAGS env var exists, the current process is // considered a child process spawned by the Centipede binary as the runner, // and we should not run CentipedeMain in this process. const bool runner_mode = getenv("CENTIPEDE_RUNNER_FLAGS"); + const bool is_running_property_function_in_this_process = + runner_mode || configuration.crashing_input_to_reproduce.has_value() || + getenv("FUZZTEST_REPLAY") || getenv("FUZZTEST_MINIMIZE_REPRODUCER"); + if (!is_running_property_function_in_this_process && + runtime_.termination_requested()) { + absl::FPrintF(GetStderr(), + "[.] Skipping %s since termination was requested.\n", + test_.full_name()); + runtime_.SetSkippingRequested(true); + return true; + } + runtime_.SetRunMode(mode); + runtime_.SetSkippingRequested(false); + runtime_.SetCurrentTest(&test_, &configuration); + if (is_running_property_function_in_this_process) { + if (IsSilenceTargetEnabled()) SilenceTargetStdoutAndStderr(); + runtime_.EnableReporter(&fuzzer_impl_.stats_, [] { return absl::Now(); }); + } + fuzzer_impl_.fixture_driver_->SetUpFuzzTest(); const int result = ([&]() { if (runtime_.skipping_requested()) { absl::FPrintF(GetStderr(), @@ -554,24 +571,30 @@ int CentipedeFuzzerAdaptor::RunInFuzzingMode( if (runner_mode) { CentipedeAdaptorRunnerCallbacks runner_callbacks(&runtime_, &fuzzer_impl_, &configuration); - print_final_stats = false; - return centipede::RunnerMain(argc != nullptr ? *argc : 0, - argv != nullptr ? *argv : nullptr, + static char fake_argv0[] = "fake_argv"; + static char* fake_argv[] = {fake_argv0, nullptr}; + return centipede::RunnerMain(argc != nullptr ? *argc : 1, + argv != nullptr ? *argv : fake_argv, runner_callbacks); } // Centipede engine does not support replay and reproducer minimization // (within the single process). So use the existing fuzztest implementation. // This is fine because it does not require coverage instrumentation. if (fuzzer_impl_.ReplayInputsIfAvailable(configuration)) return 0; + // `ReplayInputsIfAvailable` overwrites the run mode - revert it back. + runtime_.SetRunMode(mode); // Run as the fuzzing engine. - TempDir workdir("/tmp/fuzztest-workdir-"); + std::unique_ptr workdir; + if (configuration.corpus_database.empty() || mode == RunMode::kUnitTest) + workdir = std::make_unique("/tmp/fuzztest-workdir-"); + const std::string workdir_path = workdir ? workdir->path() : ""; const auto env = CreateCentipedeEnvironmentFromFuzzTestFlags( - configuration, workdir.path(), test_.full_name()); - CentipedeAdaptorEngineCallbacksFactory factory(&runtime_, &fuzzer_impl_, - &configuration); - if (const char* minimize_dir_chars = - std::getenv("FUZZTEST_MINIMIZE_TESTSUITE_DIR")) { - print_final_stats = false; + configuration, workdir_path, test_.full_name(), mode); + centipede::DefaultCallbacksFactory + factory; + if (!configuration.corpus_database.empty()) { + } else if (const char* minimize_dir_chars = + std::getenv("FUZZTEST_MINIMIZE_TESTSUITE_DIR")) { const std::string minimize_dir = minimize_dir_chars; const char* corpus_out_dir_chars = std::getenv("FUZZTEST_TESTSUITE_OUT_DIR"); @@ -623,13 +646,7 @@ int CentipedeFuzzerAdaptor::RunInFuzzingMode( return centipede::CentipedeMain(env, factory); })(); fuzzer_impl_.fixture_driver_->TearDownFuzzTest(); - if (result) std::exit(result); - if (print_final_stats) { - absl::FPrintF(GetStderr(), "\n[.] Fuzzing was terminated.\n"); - runtime_.PrintFinalStatsOnDefaultSink(); - absl::FPrintF(GetStderr(), "\n"); - } - return 0; + return result == 0; } } // namespace fuzztest::internal @@ -670,6 +687,7 @@ extern "C" const char* CentipedeGetRunnerFlags() { // Runner mode. Use the existing flags. return strdup(runner_flags_env); } + // Set the runner flags according to the FuzzTest default environment. const auto env = fuzztest::internal::CreateDefaultCentipedeEnvironment(); CentipedeCallbacksForRunnerFlagsExtraction callbacks(env); diff --git a/fuzztest/internal/centipede_adaptor.h b/fuzztest/internal/centipede_adaptor.h index f64bd652..295574fb 100644 --- a/fuzztest/internal/centipede_adaptor.h +++ b/fuzztest/internal/centipede_adaptor.h @@ -30,11 +30,14 @@ class CentipedeFuzzerAdaptor : public FuzzTestFuzzer { public: CentipedeFuzzerAdaptor(const FuzzTest& test, std::unique_ptr fixture_driver); - void RunInUnitTestMode(const Configuration& configuration) override; - int RunInFuzzingMode(int* argc, char*** argv, - const Configuration& configuration) override; + bool RunInUnitTestMode(const Configuration& configuration) override; + bool RunInFuzzingMode(int* argc, char*** argv, + const Configuration& configuration) override; private: + bool Run(int* argc, char*** argv, RunMode mode, + const Configuration& configuration); + Runtime& runtime_ = Runtime::instance(); const FuzzTest& test_; CentipedeFixtureDriver* centipede_fixture_driver_; diff --git a/fuzztest/internal/configuration.cc b/fuzztest/internal/configuration.cc index 31f35574..88a009ec 100644 --- a/fuzztest/internal/configuration.cc +++ b/fuzztest/internal/configuration.cc @@ -206,7 +206,8 @@ std::string Configuration::Serialize() const { SpaceFor(binary_identifier) + SpaceFor(fuzz_tests) + SpaceFor(fuzz_tests_in_current_shard) + SpaceFor(reproduce_findings_as_separate_tests) + - SpaceFor(only_replay_corpus) + SpaceFor(stack_limit) + + SpaceFor(replay_corpus) + SpaceFor(only_replay) + + SpaceFor(print_subprocess_log) + SpaceFor(stack_limit) + SpaceFor(rss_limit) + SpaceFor(time_limit_per_input_str) + SpaceFor(time_limit_str) + SpaceFor(time_budget_type_str) + SpaceFor(jobs) + SpaceFor(crashing_input_to_reproduce) + @@ -218,7 +219,9 @@ std::string Configuration::Serialize() const { offset = WriteVectorOfStrings(out, offset, fuzz_tests); offset = WriteVectorOfStrings(out, offset, fuzz_tests_in_current_shard); offset = WriteIntegral(out, offset, reproduce_findings_as_separate_tests); - offset = WriteIntegral(out, offset, only_replay_corpus); + offset = WriteIntegral(out, offset, replay_corpus); + offset = WriteIntegral(out, offset, only_replay); + offset = WriteIntegral(out, offset, print_subprocess_log); offset = WriteIntegral(out, offset, stack_limit); offset = WriteIntegral(out, offset, rss_limit); offset = WriteString(out, offset, time_limit_per_input_str); @@ -242,7 +245,9 @@ absl::StatusOr Configuration::Deserialize( ConsumeVectorOfStrings(serialized)); ASSIGN_OR_RETURN(reproduce_findings_as_separate_tests, Consume(serialized)); - ASSIGN_OR_RETURN(only_replay_corpus, Consume(serialized)); + ASSIGN_OR_RETURN(replay_corpus, Consume(serialized)); + ASSIGN_OR_RETURN(only_replay, Consume(serialized)); + ASSIGN_OR_RETURN(print_subprocess_log, Consume(serialized)); ASSIGN_OR_RETURN(stack_limit, Consume(serialized)); ASSIGN_OR_RETURN(rss_limit, Consume(serialized)); ASSIGN_OR_RETURN(time_limit_per_input_str, ConsumeString(serialized)); @@ -268,7 +273,9 @@ absl::StatusOr Configuration::Deserialize( *std::move(fuzz_tests), *std::move(fuzz_tests_in_current_shard), *reproduce_findings_as_separate_tests, - *only_replay_corpus, + *replay_corpus, + *only_replay, + *print_subprocess_log, *stack_limit, *rss_limit, *time_limit_per_input, diff --git a/fuzztest/internal/configuration.h b/fuzztest/internal/configuration.h index cccf5387..e9d987ad 100644 --- a/fuzztest/internal/configuration.h +++ b/fuzztest/internal/configuration.h @@ -59,8 +59,12 @@ struct Configuration { // Generate separate TESTs that replay crashing inputs for the selected fuzz // tests. bool reproduce_findings_as_separate_tests = false; - // Do not fuzz, only replay the corpus. - bool only_replay_corpus = false; + // If set, replay corpus inputs from the database. + bool replay_corpus = false; + // If set, replay regression/corpus inputs without fuzzing. + bool only_replay = false; + // If set, print log from subprocesses spawned by FuzzTest. + bool print_subprocess_log = false; // Stack limit in bytes. size_t stack_limit = 128 * 1024; diff --git a/fuzztest/internal/configuration_test.cc b/fuzztest/internal/configuration_test.cc index 4ce02a1a..789a6f2a 100644 --- a/fuzztest/internal/configuration_test.cc +++ b/fuzztest/internal/configuration_test.cc @@ -21,7 +21,9 @@ MATCHER_P(IsOkAndEquals, config, "") { other->fuzz_tests_in_current_shard && config.reproduce_findings_as_separate_tests == other->reproduce_findings_as_separate_tests && - config.only_replay_corpus == other->only_replay_corpus && + config.replay_corpus == other->replay_corpus && + config.only_replay == other->only_replay && + config.print_subprocess_log == other->print_subprocess_log && config.stack_limit == other->stack_limit && config.rss_limit == other->rss_limit && config.time_limit_per_input == other->time_limit_per_input && @@ -42,7 +44,9 @@ TEST(ConfigurationTest, /*fuzz_tests=*/{}, /*fuzz_tests_in_current_shard=*/{}, /*reproduce_findings_as_separate_tests=*/true, - /*only_replay_corpus=*/true, + /*replay_corpus=*/true, + /*only_replay=*/true, + /*print_subprocess_log=*/true, /*stack_limit=*/100, /*rss_limit=*/200, /*time_limit_per_input=*/absl::Seconds(42), @@ -64,7 +68,9 @@ TEST(ConfigurationTest, {"FuzzTest1", "FuzzTest2"}, {"FuzzTest1"}, /*reproduce_findings_as_separate_tests=*/true, - /*only_replay_corpus=*/true, + /*replay_corpus=*/true, + /*only_replay=*/true, + /*print_subprocess_log=*/true, /*stack_limit=*/100, /*rss_limit=*/200, /*time_limit_per_input=*/absl::Seconds(42), diff --git a/fuzztest/internal/googletest_adaptor.h b/fuzztest/internal/googletest_adaptor.h index 7286a449..3e7f2d4e 100644 --- a/fuzztest/internal/googletest_adaptor.h +++ b/fuzztest/internal/googletest_adaptor.h @@ -78,13 +78,14 @@ class GTest_TestAdaptor : public ::testing::Test { EXPECT_TRUE(false) << "Death test is not supported."; #endif } else { - test->RunInUnitTestMode(configuration_); + EXPECT_TRUE(test->RunInUnitTestMode(configuration_)) + << "Failure(s) found in the unit-test mode."; } } else { // TODO(b/245753736): Consider using `tolerate_failure` when FuzzTest can // tolerate crashes in fuzzing mode. - ASSERT_EQ(0, test->RunInFuzzingMode(argc_, argv_, configuration_)) - << "Fuzzing failure."; + EXPECT_TRUE(test->RunInFuzzingMode(argc_, argv_, configuration_)) + << "Failure(s) found in the fuzzing mode."; } } diff --git a/fuzztest/internal/runtime.cc b/fuzztest/internal/runtime.cc index ecdf0200..c9ea6a95 100644 --- a/fuzztest/internal/runtime.cc +++ b/fuzztest/internal/runtime.cc @@ -20,6 +20,7 @@ #endif #include +#include #include #include #include @@ -933,7 +934,7 @@ void PopulateLimits(const Configuration& configuration, #endif } -void FuzzTestFuzzerImpl::RunInUnitTestMode(const Configuration& configuration) { +bool FuzzTestFuzzerImpl::RunInUnitTestMode(const Configuration& configuration) { runtime_.SetSkippingRequested(false); fixture_driver_->SetUpFuzzTest(); [&] { @@ -1014,6 +1015,7 @@ void FuzzTestFuzzerImpl::RunInUnitTestMode(const Configuration& configuration) { runtime_.SetCurrentTest(nullptr, nullptr); }(); fixture_driver_->TearDownFuzzTest(); + return true; } FuzzTestFuzzerImpl::RunResult FuzzTestFuzzerImpl::RunOneInput( @@ -1102,16 +1104,16 @@ void FuzzTestFuzzerImpl::MinimizeNonFatalFailureLocally(absl::BitGenRef prng) { } } -int FuzzTestFuzzerImpl::RunInFuzzingMode(int* /*argc*/, char*** /*argv*/, - const Configuration& configuration) { +bool FuzzTestFuzzerImpl::RunInFuzzingMode(int* /*argc*/, char*** /*argv*/, + const Configuration& configuration) { runtime_.SetSkippingRequested(false); fixture_driver_->SetUpFuzzTest(); - const int exit_code = [&] { + const bool success = [&] { if (runtime_.skipping_requested()) { absl::FPrintF(GetStderr(), "[.] Skipping %s per request from the test setup.\n", test_.full_name()); - return 0; + return true; } runtime_.StartWatchdog(); PopulateLimits(configuration, execution_coverage_); @@ -1125,14 +1127,14 @@ int FuzzTestFuzzerImpl::RunInFuzzingMode(int* /*argc*/, char*** /*argv*/, if (ReplayInputsIfAvailable(configuration)) { // If ReplayInputs returns, it means the replay didn't crash. // We don't want to actually run the fuzzer so exit now. - return 0; + return true; } if (execution_coverage_ == nullptr) { absl::FPrintF( GetStderr(), "\n\n[!] To fuzz, please build with --config=fuzztest.\n\n\n"); - return 1; + return false; } stats_.total_edges = execution_coverage_->GetCounterMap().size(); @@ -1144,7 +1146,7 @@ int FuzzTestFuzzerImpl::RunInFuzzingMode(int* /*argc*/, char*** /*argv*/, GetStderr(), "[*] Selected %d corpus inputs in minimization mode - exiting.\n", stats_.useful_inputs); - return 0; + return true; } CorpusDatabase corpus_database(configuration); @@ -1238,10 +1240,10 @@ int FuzzTestFuzzerImpl::RunInFuzzingMode(int* /*argc*/, char*** /*argv*/, absl::FPrintF(GetStderr(), "\n[.] Fuzzing was terminated.\n"); runtime_.PrintFinalStatsOnDefaultSink(); absl::FPrintF(GetStderr(), "\n"); - return 0; + return true; }(); fixture_driver_->TearDownFuzzTest(); - return exit_code; + return success; } } // namespace fuzztest::internal diff --git a/fuzztest/internal/runtime.h b/fuzztest/internal/runtime.h index 89fd494a..491fe704 100644 --- a/fuzztest/internal/runtime.h +++ b/fuzztest/internal/runtime.h @@ -63,10 +63,11 @@ namespace internal { class FuzzTestFuzzer { public: virtual ~FuzzTestFuzzer() = default; - virtual void RunInUnitTestMode(const Configuration& configuration) = 0; - // Returns fuzzing mode's exit code. Zero indicates success. - virtual int RunInFuzzingMode(int* argc, char*** argv, - const Configuration& configuration) = 0; + // Returns ture if no error were detected by the FuzzTest, false otherwise. + virtual bool RunInUnitTestMode(const Configuration& configuration) = 0; + // Returns ture if no error were detected by the FuzzTest, false otherwise. + virtual bool RunInFuzzingMode(int* argc, char*** argv, + const Configuration& configuration) = 0; }; class FuzzTest; @@ -280,11 +281,11 @@ class FuzzTestFuzzerImpl : public FuzzTestFuzzer { private: // TODO(fniksic): Refactor to reduce code complexity and improve readability. - void RunInUnitTestMode(const Configuration& configuration) override; + bool RunInUnitTestMode(const Configuration& configuration) override; // TODO(fniksic): Refactor to reduce code complexity and improve readability. - int RunInFuzzingMode(int* argc, char*** argv, - const Configuration& configuration) override; + bool RunInFuzzingMode(int* argc, char*** argv, + const Configuration& configuration) override; // Use the standard PRNG instead of absl::BitGen because Abseil doesn't // guarantee seed stability