Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rewrite a few tests to prepare for the unified execution model. #1483

Merged
merged 1 commit into from
Dec 17, 2024
Merged

Rewrite a few tests to prepare for the unified execution model. #1483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 45 additions & 44 deletions e2e_tests/functional_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,22 @@ absl::flat_hash_map<std::string, std::string> WithTestSanitizerOptions(
return env;
}

int CountSubstrs(absl::string_view haystack, absl::string_view needle) {
int count = 0;
while (true) {
size_t pos = haystack.find(needle);
if (pos == haystack.npos) return count;
++count;
haystack.remove_prefix(pos + needle.size());
}
}

// Counts the number of times the target binary has been run. Needed because
// Centipede runs the binary multiple times.
int CountTargetRuns(absl::string_view std_err) {
return CountSubstrs(std_err, "FuzzTest functional test target run");
}

class UnitTestModeTest : public ::testing::Test {
protected:
RunResults Run(
Expand Down Expand Up @@ -128,16 +144,6 @@ TEST_F(UnitTestModeTest, UnitTestModeLimitsNumberOfIterationsByWallTime) {
EXPECT_THAT(status, Eq(ExitCode(0)));
}

int CountSubstrs(absl::string_view haystack, absl::string_view needle) {
int count = 0;
while (true) {
size_t pos = haystack.find(needle);
if (pos == haystack.npos) return count;
++count;
haystack.remove_prefix(pos + needle.size());
}
}

RE2 MakeReproducerRegex(absl::string_view suite_name,
absl::string_view test_name, absl::string_view args) {
return RE2(
Expand Down Expand Up @@ -207,18 +213,21 @@ TEST_F(UnitTestModeTest,

TEST_F(UnitTestModeTest, GlobalEnvironmentGoesThroughCompleteLifecycle) {
auto [status, std_out, std_err] = Run("MySuite.GoogleTestExpect");
EXPECT_GT(CountSubstrs(std_err, "<<GlobalEnvironment::GlobalEnvironment()>>"),
0);
EXPECT_EQ(
1, CountSubstrs(std_err, "<<GlobalEnvironment::GlobalEnvironment()>>"));
EXPECT_EQ(1, CountSubstrs(std_err, "<<GlobalEnvironment::SetUp()>>"));
EXPECT_EQ(1, CountSubstrs(std_err, "<<GlobalEnvironment::TearDown()>>"));
EXPECT_EQ(
1, CountSubstrs(std_err, "<<GlobalEnvironment::~GlobalEnvironment()>>"));
CountSubstrs(std_err, "<<GlobalEnvironment::GlobalEnvironment()>>"),
CountSubstrs(std_err, "<<GlobalEnvironment::~GlobalEnvironment()>>"));
EXPECT_GT(CountSubstrs(std_err, "<<GlobalEnvironment::SetUp()>>"), 0);
EXPECT_EQ(CountSubstrs(std_err, "<<GlobalEnvironment::SetUp()>>"),
CountSubstrs(std_err, "<<GlobalEnvironment::TearDown()>>"));
}

TEST_F(UnitTestModeTest, FixtureGoesThroughCompleteLifecycle) {
auto [status, std_out, std_err] = Run("FixtureTest.NeverFails");
EXPECT_EQ(1, CountSubstrs(std_err, "<<FixtureTest::FixtureTest()>>"));
EXPECT_EQ(1, CountSubstrs(std_err, "<<FixtureTest::~FixtureTest()>>"));
EXPECT_GT(CountSubstrs(std_err, "<<FixtureTest::FixtureTest()>>"), 0);
EXPECT_EQ(CountSubstrs(std_err, "<<FixtureTest::FixtureTest()>>"),
CountSubstrs(std_err, "<<FixtureTest::~FixtureTest()>>"));
}

TEST_F(UnitTestModeTest,
Expand All @@ -231,19 +240,19 @@ TEST_F(UnitTestModeTest,
TEST_F(UnitTestModeTest,
GoogleTestPerFuzzTestFixtureInstantiatedOncePerFuzzTest) {
auto [status, std_out, std_err] =
Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen");
EXPECT_EQ(
1, CountSubstrs(std_err, "<<CallCountGoogleTest::call_count_ == 10>>"));
Run("CallCountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount");
EXPECT_THAT(status, Eq(ExitCode(0)));
}

TEST_F(UnitTestModeTest, GoogleTestStaticTestSuiteFunctionsCalledOnce) {
TEST_F(UnitTestModeTest, GoogleTestStaticTestSuiteFunctionsCalledInBalance) {
auto [status, std_out, std_err] =
Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen:"
Run("CallCountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount:"
"CallCountPerFuzzTest.NeverFails");
EXPECT_EQ(1,
CountSubstrs(std_err, "<<CallCountGoogleTest::SetUpTestSuite()>>"));
EXPECT_GT(CountSubstrs(std_err, "<<CallCountGoogleTest::SetUpTestSuite()>>"),
0);
EXPECT_EQ(
1, CountSubstrs(std_err, "<<CallCountGoogleTest::TearDownTestSuite()>>"));
CountSubstrs(std_err, "<<CallCountGoogleTest::SetUpTestSuite()>>"),
CountSubstrs(std_err, "<<CallCountGoogleTest::TearDownTestSuite()>>"));
}

TEST_F(UnitTestModeTest, GoogleTestWorksWithProtoExtensionsUsedInSeeds) {
Expand Down Expand Up @@ -769,10 +778,10 @@ TEST_F(FuzzingModeCommandLineInterfaceTest, ReproducerIsDumpedWhenEnvVarIsSet) {
auto args = parsed->ToCorpus<std::tuple<std::string>>();
EXPECT_THAT(args, Optional(FieldsAre(StartsWith("Fuzz")))) << std_err;
EXPECT_THAT(std_err,
HasSubstr(absl::StrCat("Reproducer file was dumped at:\n",
replay_files[0].path)));
EXPECT_THAT(std_err, HasSubstr(absl::StrCat("--test_env=FUZZTEST_REPLAY=",
replay_files[0].path)));
AllOf(HasSubstr("Reproducer file was dumped at:"),
HasSubstr(replay_files[0].path),
HasSubstr(absl::StrCat("--test_env=FUZZTEST_REPLAY=",
replay_files[0].path))));
}

TEST_F(FuzzingModeCommandLineInterfaceTest, SavesCorpusWhenEnvVarIsSet) {
Expand Down Expand Up @@ -991,8 +1000,8 @@ TEST_F(FuzzingModeCommandLineInterfaceTest,
ReplayFile replay(std::in_place, std::tuple<uint8_t, double>{10, 1979.125});
auto [status, std_out, std_err] =
RunWith({{"fuzz", "MySuite.WithDomainClass"}}, replay.GetReplayEnv());
EXPECT_THAT(std_err, HasSubstr("argument 0: 10"));
EXPECT_THAT(std_err, HasSubstr("argument 1: 1979.125"));
EXPECT_THAT(std_err, HasSubstr("argument 0: 10")) << std_err;
EXPECT_THAT(std_err, HasSubstr("argument 1: 1979.125")) << std_err;
EXPECT_THAT(status, Ne(ExitCode(0)));
}

Expand Down Expand Up @@ -1253,16 +1262,6 @@ class FuzzingModeFixtureTest
.timeout = absl::InfiniteDuration()});
}
}

// Counts the number of times the target binary has been run. Needed because
// Centipede runs the binary multiple times.
int CountTargetRuns(absl::string_view std_err) {
if (GetParam().multi_process) {
return CountSubstrs(std_err, "Centipede fuzz target runner; argv[0]:");
} else {
return 1;
}
}
};

TEST_P(FuzzingModeFixtureTest, GlobalEnvironmentIsSetUpForFailingTest) {
Expand Down Expand Up @@ -1315,13 +1314,15 @@ TEST_P(FuzzingModeFixtureTest,
TEST_P(FuzzingModeFixtureTest,
GoogleTestPerFuzzTestFixtureInstantiatedOncePerFuzzTest) {
auto [status, std_out, std_err] =
Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen", /*iterations=*/10);
EXPECT_THAT(std_err, HasSubstr("<<CallCountGoogleTest::call_count_ == 10>>"));
Run("CallCountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount",
/*iterations=*/10);
EXPECT_THAT(status, Eq(ExitCode(0)));
}

TEST_P(FuzzingModeFixtureTest, GoogleTestStaticTestSuiteFunctionsCalledOnce) {
auto [status, std_out, std_err] =
Run("CallCountPerFuzzTest.CallCountReachesAtLeastTen", /*iterations=*/10);
Run("CountPerFuzzTest.CallCountPerFuzzTestEqualsToGlobalCount",
/*iterations=*/10);
EXPECT_GT(CountTargetRuns(std_err), 0);
EXPECT_EQ(CountTargetRuns(std_err),
CountSubstrs(std_err, "<<CallCountGoogleTest::SetUpTestSuite()>>"));
Expand Down
16 changes: 13 additions & 3 deletions e2e_tests/testdata/fuzz_tests_for_functional_testing.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ using ::fuzztest::internal::TestProtobufWithRequired;
using ::fuzztest::internal::TestSubProtobuf;
using ::google::protobuf::FieldDescriptor;

bool print_target_run_message_once = []() {
fputs("FuzzTest functional test target run\n", stderr);
return true;
}();

void PassesWithPositiveInput(int x) {
if (x <= 0) std::abort();
}
Expand Down Expand Up @@ -757,12 +762,17 @@ FUZZ_TEST_F(AlternateSignalStackFixture,

void DetectRegressionAndCoverageInputs(const std::string& input) {
if (absl::StartsWith(input, "regression")) {
std::cout << "regression input detected: " << input << std::endl;
std::cerr << "regression input detected: " << input << std::endl;
}
if (absl::StartsWith(input, "coverage")) {
std::cout << "coverage input detected: " << input << std::endl;
std::cerr << "coverage input detected: " << input << std::endl;
// Sleep for the first coverage input for depleting the replay time budget.
static bool first_input = true;
if (first_input) {
first_input = false;
absl::SleepFor(absl::Seconds(2));
}
}
absl::SleepFor(absl::Seconds(0.1));
}
FUZZ_TEST(MySuite, DetectRegressionAndCoverageInputs);

Expand Down
15 changes: 7 additions & 8 deletions e2e_tests/testdata/fuzz_tests_using_googletest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ class CallCountGoogleTest : public testing::Test {
fprintf(stderr, "<<CallCountGoogleTest::TearDownTestSuite()>>\n");
}

int call_count_ = 0;
unsigned int call_count_ = 0;
};

class CallCountPerIteration
Expand All @@ -88,19 +88,18 @@ class CallCountPerIteration
FUZZ_TEST_F(CallCountPerIteration,
CallCountIsAlwaysIncrementedFromInitialValue);

static unsigned int global_call_count = 0;
class CallCountPerFuzzTest
: public ::fuzztest::PerFuzzTestFixtureAdapter<CallCountGoogleTest> {
public:
void CallCountReachesAtLeastTen(int) {
if (call_count_ < std::numeric_limits<int>::max()) ++call_count_;
if (call_count_ == 10) {
fprintf(stderr, "<<CallCountGoogleTest::call_count_ == %d>>\n",
call_count_);
}
void CallCountPerFuzzTestEqualsToGlobalCount(int) {
++call_count_;
++global_call_count;
EXPECT_EQ(call_count_, global_call_count);
}
void NeverFails(int) {}
};
FUZZ_TEST_F(CallCountPerFuzzTest, CallCountReachesAtLeastTen);
FUZZ_TEST_F(CallCountPerFuzzTest, CallCountPerFuzzTestEqualsToGlobalCount);
FUZZ_TEST_F(CallCountPerFuzzTest, NeverFails);

TEST(SharedSuite, WorksAsUnitTest) {}
Expand Down
Loading