diff --git a/e2e_tests/functional_test.cc b/e2e_tests/functional_test.cc index 33225640..a5ccbbf4 100644 --- a/e2e_tests/functional_test.cc +++ b/e2e_tests/functional_test.cc @@ -75,6 +75,22 @@ absl::flat_hash_map WithTestSanitizerOptions( return env; } +int CountSubstrs(absl::string_view haystack, absl::string_view needle) { + int count = 0; + while (true) { + size_t pos = haystack.find(needle); + if (pos == haystack.npos) return count; + ++count; + haystack.remove_prefix(pos + needle.size()); + } +} + +// Counts the number of times the target binary has been run. Needed because +// Centipede runs the binary multiple times. +int CountTargetRuns(absl::string_view std_err) { + return CountSubstrs(std_err, "FuzzTest functional test target run"); +} + class UnitTestModeTest : public ::testing::Test { protected: RunResults Run( @@ -128,16 +144,6 @@ TEST_F(UnitTestModeTest, UnitTestModeLimitsNumberOfIterationsByWallTime) { EXPECT_THAT(status, Eq(ExitCode(0))); } -int CountSubstrs(absl::string_view haystack, absl::string_view needle) { - int count = 0; - while (true) { - size_t pos = haystack.find(needle); - if (pos == haystack.npos) return count; - ++count; - haystack.remove_prefix(pos + needle.size()); - } -} - RE2 MakeReproducerRegex(absl::string_view suite_name, absl::string_view test_name, absl::string_view args) { return RE2( @@ -207,18 +213,21 @@ TEST_F(UnitTestModeTest, TEST_F(UnitTestModeTest, GlobalEnvironmentGoesThroughCompleteLifecycle) { auto [status, std_out, std_err] = Run("MySuite.GoogleTestExpect"); + EXPECT_GT(CountSubstrs(std_err, "<>"), + 0); EXPECT_EQ( - 1, CountSubstrs(std_err, "<>")); - EXPECT_EQ(1, CountSubstrs(std_err, "<>")); - EXPECT_EQ(1, CountSubstrs(std_err, "<>")); - EXPECT_EQ( - 1, CountSubstrs(std_err, "<>")); + CountSubstrs(std_err, "<>"), + CountSubstrs(std_err, "<>")); + EXPECT_GT(CountSubstrs(std_err, "<>"), 0); + EXPECT_EQ(CountSubstrs(std_err, "<>"), + CountSubstrs(std_err, "<>")); } TEST_F(UnitTestModeTest, FixtureGoesThroughCompleteLifecycle) { auto [status, std_out, std_err] = Run("FixtureTest.NeverFails"); - EXPECT_EQ(1, CountSubstrs(std_err, "<>")); - EXPECT_EQ(1, CountSubstrs(std_err, "<>")); + EXPECT_GT(CountSubstrs(std_err, "<>"), 0); + EXPECT_EQ(CountSubstrs(std_err, "<>"), + CountSubstrs(std_err, "<>")); } TEST_F(UnitTestModeTest, @@ -231,19 +240,19 @@ TEST_F(UnitTestModeTest, TEST_F(UnitTestModeTest, GoogleTestPerFuzzTestFixtureInstantiatedOncePerFuzzTest) { auto [status, std_out, std_err] = - Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen"); - EXPECT_EQ( - 1, CountSubstrs(std_err, "<>")); + Run("CallCountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount"); + EXPECT_THAT(status, Eq(ExitCode(0))); } -TEST_F(UnitTestModeTest, GoogleTestStaticTestSuiteFunctionsCalledOnce) { +TEST_F(UnitTestModeTest, GoogleTestStaticTestSuiteFunctionsCalledInBalance) { auto [status, std_out, std_err] = - Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen:" + Run("CallCountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount:" "CallCountPerFuzzTest.NeverFails"); - EXPECT_EQ(1, - CountSubstrs(std_err, "<>")); + EXPECT_GT(CountSubstrs(std_err, "<>"), + 0); EXPECT_EQ( - 1, CountSubstrs(std_err, "<>")); + CountSubstrs(std_err, "<>"), + CountSubstrs(std_err, "<>")); } TEST_F(UnitTestModeTest, GoogleTestWorksWithProtoExtensionsUsedInSeeds) { @@ -769,10 +778,10 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, ReproducerIsDumpedWhenEnvVarIsSet) { auto args = parsed->ToCorpus>(); EXPECT_THAT(args, Optional(FieldsAre(StartsWith("Fuzz")))) << std_err; EXPECT_THAT(std_err, - HasSubstr(absl::StrCat("Reproducer file was dumped at:\n", - replay_files[0].path))); - EXPECT_THAT(std_err, HasSubstr(absl::StrCat("--test_env=FUZZTEST_REPLAY=", - replay_files[0].path))); + AllOf(HasSubstr("Reproducer file was dumped at:"), + HasSubstr(replay_files[0].path), + HasSubstr(absl::StrCat("--test_env=FUZZTEST_REPLAY=", + replay_files[0].path)))); } TEST_F(FuzzingModeCommandLineInterfaceTest, SavesCorpusWhenEnvVarIsSet) { @@ -991,8 +1000,8 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, ReplayFile replay(std::in_place, std::tuple{10, 1979.125}); auto [status, std_out, std_err] = RunWith({{"fuzz", "MySuite.WithDomainClass"}}, replay.GetReplayEnv()); - EXPECT_THAT(std_err, HasSubstr("argument 0: 10")); - EXPECT_THAT(std_err, HasSubstr("argument 1: 1979.125")); + EXPECT_THAT(std_err, HasSubstr("argument 0: 10")) << std_err; + EXPECT_THAT(std_err, HasSubstr("argument 1: 1979.125")) << std_err; EXPECT_THAT(status, Ne(ExitCode(0))); } @@ -1253,16 +1262,6 @@ class FuzzingModeFixtureTest .timeout = absl::InfiniteDuration()}); } } - - // Counts the number of times the target binary has been run. Needed because - // Centipede runs the binary multiple times. - int CountTargetRuns(absl::string_view std_err) { - if (GetParam().multi_process) { - return CountSubstrs(std_err, "Centipede fuzz target runner; argv[0]:"); - } else { - return 1; - } - } }; TEST_P(FuzzingModeFixtureTest, GlobalEnvironmentIsSetUpForFailingTest) { @@ -1315,13 +1314,15 @@ TEST_P(FuzzingModeFixtureTest, TEST_P(FuzzingModeFixtureTest, GoogleTestPerFuzzTestFixtureInstantiatedOncePerFuzzTest) { auto [status, std_out, std_err] = - Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen", /*iterations=*/10); - EXPECT_THAT(std_err, HasSubstr("<>")); + Run("CallCountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount", + /*iterations=*/10); + EXPECT_THAT(status, Eq(ExitCode(0))); } TEST_P(FuzzingModeFixtureTest, GoogleTestStaticTestSuiteFunctionsCalledOnce) { auto [status, std_out, std_err] = - Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen", /*iterations=*/10); + Run("CountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount", + /*iterations=*/10); EXPECT_GT(CountTargetRuns(std_err), 0); EXPECT_EQ(CountTargetRuns(std_err), CountSubstrs(std_err, "<>")); diff --git a/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc b/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc index 50a1abfb..eeec09ae 100644 --- a/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc +++ b/e2e_tests/testdata/fuzz_tests_for_functional_testing.cc @@ -65,6 +65,11 @@ using ::fuzztest::internal::TestProtobufWithRequired; using ::fuzztest::internal::TestSubProtobuf; using ::google::protobuf::FieldDescriptor; +bool print_target_run_message_once = []() { + fputs("FuzzTest functional test target run\n", stderr); + return true; +}(); + void PassesWithPositiveInput(int x) { if (x <= 0) std::abort(); } @@ -757,12 +762,17 @@ FUZZ_TEST_F(AlternateSignalStackFixture, void DetectRegressionAndCoverageInputs(const std::string& input) { if (absl::StartsWith(input, "regression")) { - std::cout << "regression input detected: " << input << std::endl; + std::cerr << "regression input detected: " << input << std::endl; } if (absl::StartsWith(input, "coverage")) { - std::cout << "coverage input detected: " << input << std::endl; + std::cerr << "coverage input detected: " << input << std::endl; + // Sleep for the first coverage input for depleting the replay time budget. + static bool first_input = true; + if (first_input) { + first_input = false; + absl::SleepFor(absl::Seconds(2)); + } } - absl::SleepFor(absl::Seconds(0.1)); } FUZZ_TEST(MySuite, DetectRegressionAndCoverageInputs); diff --git a/e2e_tests/testdata/fuzz_tests_using_googletest.cc b/e2e_tests/testdata/fuzz_tests_using_googletest.cc index d2d01caf..9fcb0b10 100644 --- a/e2e_tests/testdata/fuzz_tests_using_googletest.cc +++ b/e2e_tests/testdata/fuzz_tests_using_googletest.cc @@ -75,7 +75,7 @@ class CallCountGoogleTest : public testing::Test { fprintf(stderr, "<>\n"); } - int call_count_ = 0; + unsigned int call_count_ = 0; }; class CallCountPerIteration @@ -88,19 +88,18 @@ class CallCountPerIteration FUZZ_TEST_F(CallCountPerIteration, CallCountIsAlwaysIncrementedFromInitialValue); +static unsigned int global_call_count = 0; class CallCountPerFuzzTest : public ::fuzztest::PerFuzzTestFixtureAdapter { public: - void CallCountReachesAtLeastTen(int) { - if (call_count_ < std::numeric_limits::max()) ++call_count_; - if (call_count_ == 10) { - fprintf(stderr, "<>\n", - call_count_); - } + void CallCountPerFuzzTestEqualsToGlobalCount(int) { + ++call_count_; + ++global_call_count; + EXPECT_EQ(call_count_, global_call_count); } void NeverFails(int) {} }; -FUZZ_TEST_F(CallCountPerFuzzTest, CallCountReachesAtLeastTen); +FUZZ_TEST_F(CallCountPerFuzzTest, CallCountPerFuzzTestEqualsToGlobalCount); FUZZ_TEST_F(CallCountPerFuzzTest, NeverFails); TEST(SharedSuite, WorksAsUnitTest) {}