Add support for testing with stdin

explorer/file_test.cpp

@@ -29,7 +29,8 @@ class ExplorerFileTest : public FileTestBase {
 
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
            llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
-           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+           FILE* /*input_stream*/, llvm::raw_pwrite_stream& output_stream,
+           llvm::raw_pwrite_stream& error_stream)
       -> ErrorOr<RunResult> override {
     // Add the prelude.
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> prelude =
@@ -52,9 +53,10 @@ class ExplorerFileTest : public FileTestBase {
       args.push_back(arg.data());
     }
 
-    int exit_code = ExplorerMain(
-        args.size(), args.data(), /*install_path=*/"", PreludePath, stdout,
-        stderr, check_trace_output() ? stdout : trace_stream_, *fs);
+    int exit_code =
+        ExplorerMain(args.size(), args.data(), /*install_path=*/"", PreludePath,
+                     output_stream, error_stream,
+                     check_trace_output() ? output_stream : trace_stream_, *fs);
 
     return {{.success = exit_code == EXIT_SUCCESS}};
   }

testing/file_test/README.md

@@ -50,9 +50,11 @@ class MyFileTest : public FileTestBase {
   // Called as part of individual test executions.
   auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
            const llvm::SmallVector<TestFile>& test_files,
-           llvm::raw_pwrite_stream& stdout, llvm::raw_pwrite_stream& stderr)
+           FILE* input_stream, llvm::raw_pwrite_stream& output_stream,
+           llvm::raw_pwrite_stream& error_stream)
       -> ErrorOr<RunResult> override {
-    return MyFunctionality(test_args, stdout, stderr);
+    return MyFunctionality(test_args, input_stream, output_stream,
+                           error_stream);
   }
 
   // Provides arguments which are used in tests that don't provide ARGS.
@@ -108,21 +110,22 @@ Supported comment markers are:
     // NOAUTOUPDATE
     ```
 
-    Controls whether the checks in the file will be autoupdated if --autoupdate
-    is passed. Exactly one of these markers must be present. If the file uses
-    splits, the marker must currently be before any splits.
+    Controls whether the checks in the file will be autoupdated if
+    `--autoupdate` is passed. Exactly one of these markers must be present. If
+    the file uses splits, the marker must currently be before any splits.
 
-    When autoupdating, CHECKs will be inserted starting below AUTOUPDATE. When a
-    CHECK has line information, autoupdate will try to insert the CHECK
-    immediately next to the line it's associated with, with stderr CHECKs
-    preceding the line and stdout CHECKs following the line. When that happens,
-    any subsequent CHECK lines without line information, or that refer to lines
-    appearing earlier, will immediately follow. As an exception, if no STDOUT
-    check line refers to any line in the test, all STDOUT check lines are placed
-    at the end of the file instead of immediately after AUTOUPDATE.
+    When autoupdating, `CHECK`s will be inserted starting below `AUTOUPDATE`.
+    When a `CHECK` has line information, autoupdate will try to insert the
+    `CHECK` immediately next to the line it's associated with, with stderr
+    `CHECK`s preceding the line and stdout `CHECK`s following the line. When
+    that happens, any subsequent `CHECK` lines without line information, or that
+    refer to lines appearing earlier, will immediately follow. As an exception,
+    if no `STDOUT` check line refers to any line in the test, all `STDOUT` check
+    lines are placed at the end of the file instead of immediately after
+    `AUTOUPDATE`.
 
     When using split files, if the last split file is named
-    `// --- AUTOUPDATE-SPLIT`, all CHECKs will be added there; no line
+    `// --- AUTOUPDATE-SPLIT`, all `CHECK`s will be added there; no line
     associations occur.
 
 -   ```
@@ -149,8 +152,8 @@ Supported comment markers are:
         Replaces some implementation-specific identifier with a value. (Mappings
         provided by way of an optional `MyFileTest::GetArgReplacements`)
 
-    ARGS can be specified at most once. If not provided, the FileTestBase child
-    is responsible for providing default arguments.
+    `ARGS` can be specified at most once. If not provided, the `FileTestBase`
+    child is responsible for providing default arguments.
 
 -   ```
     // EXTRA-ARGS: <arguments>
@@ -173,28 +176,33 @@ Supported comment markers are:
     This should be avoided because we are partly ensuring that streams are an
     API, but is helpful when wrapping Clang, where stderr is used directly.
 
-    SET-CAPTURE-CONSOLE-OUTPUT can be specified at most once.
+    `SET-CAPTURE-CONSOLE-OUTPUT` can be specified at most once.
 
 -   ```
     // SET-CHECK-SUBSET
     ```
 
-    By default, all lines of output must have a CHECK match. Adding this as a
+    By default, all lines of output must have a `CHECK` match. Adding this as a
     option sets it so that non-matching lines are ignored. All provided
-    CHECK:STDOUT: and CHECK:STDERR: lines must still have a match in output.
+    `CHECK:STDOUT:` and `CHECK:STDERR:` lines must still have a match in output.
 
-    SET-CHECK-SUBSET can be specified at most once.
+    `SET-CHECK-SUBSET`can be specified at most once.
 
 -   ```
     // --- <filename>
     ```
 
     By default, all file content is provided to the test as a single file in
-    test_files. Using this marker allows the file to be split into multiple
-    files which will all be passed to test_files.
+    `test_files`. Using this marker allows the file to be split into multiple
+    files which will all be passed to `test_files`.
 
-    Files are not created on disk; it's expected the child will create an
-    InMemoryFilesystem if needed.
+    Files are not created on disk; instead, content is passed in through the
+    `fs` passed to `Run`.
+
+    If the filename is `STDIN`, it will be provided as `input_stream` instead of
+    in `test_files`. Currently, autoupdate can place `CHECK` lines in the
+    `STDIN` split; use `AUTOUPDATE-SPLIT` to avoid that (see `AUTOUPDATE` for
+    information).
 
 -   ```
     // CHECK:STDOUT: <output line>

testing/file_test/autoupdate.h

@@ -39,7 +39,7 @@ class FileTestAutoupdater {
       const llvm::SmallVector<llvm::StringRef>& filenames,
       int autoupdate_line_number, bool autoupdate_split,
       const llvm::SmallVector<FileTestLine>& non_check_lines,
-      llvm::StringRef stdout, llvm::StringRef stderr,
+      llvm::StringRef actual_stdout, llvm::StringRef actual_stderr,
       const std::optional<RE2>& default_file_re,
       const llvm::SmallVector<LineNumberReplacement>& line_number_replacements,
       std::function<void(std::string&)> do_extra_check_replacements)
@@ -56,8 +56,8 @@ class FileTestAutoupdater {
         do_extra_check_replacements_(std::move(do_extra_check_replacements)),
         // BuildCheckLines should only be called after other member
         // initialization.
-        stdout_(BuildCheckLines(stdout, "STDOUT")),
-        stderr_(BuildCheckLines(stderr, "STDERR")),
+        stdout_(BuildCheckLines(actual_stdout, "STDOUT")),
+        stderr_(BuildCheckLines(actual_stderr, "STDERR")),
         any_attached_stdout_lines_(llvm::any_of(
             stdout_.lines,
             [&](const CheckLine& line) { return line.line_number() != -1; })),

testing/file_test/file_test_base.cpp

@@ -58,6 +58,8 @@ using ::testing::Matcher;
 using ::testing::MatchesRegex;
 using ::testing::StrEq;
 
+static constexpr llvm::StringLiteral StdinFilename = "STDIN";
+
 // Reads a file to string.
 static auto ReadFile(std::string_view path) -> ErrorOr<std::string> {
   std::ifstream proto_file{std::string(path)};
@@ -186,15 +188,15 @@ auto FileTestBase::TestBody() -> void {
   }
   SCOPED_TRACE(update_message);
   if (context.check_subset) {
-    EXPECT_THAT(SplitOutput(context.stdout),
+    EXPECT_THAT(SplitOutput(context.actual_stdout),
                 IsSupersetOf(context.expected_stdout));
-    EXPECT_THAT(SplitOutput(context.stderr),
+    EXPECT_THAT(SplitOutput(context.actual_stderr),
                 IsSupersetOf(context.expected_stderr));
 
   } else {
-    EXPECT_THAT(SplitOutput(context.stdout),
+    EXPECT_THAT(SplitOutput(context.actual_stdout),
                 ElementsAreArray(context.expected_stdout));
-    EXPECT_THAT(SplitOutput(context.stderr),
+    EXPECT_THAT(SplitOutput(context.actual_stderr),
                 ElementsAreArray(context.expected_stderr));
   }
 
@@ -232,8 +234,9 @@ auto FileTestBase::RunAutoupdater(const TestContext& context, bool dry_run)
              GetBazelCommand(BazelMode::Test, test_name_),
              GetBazelCommand(BazelMode::Dump, test_name_),
              context.input_content, filenames, *context.autoupdate_line_number,
-             context.autoupdate_split, context.non_check_lines, context.stdout,
-             context.stderr, GetDefaultFileRE(expected_filenames),
+             context.autoupdate_split, context.non_check_lines,
+             context.actual_stdout, context.actual_stderr,
+             GetDefaultFileRE(expected_filenames),
              GetLineNumberReplacements(expected_filenames),
              [&](std::string& line) { DoExtraCheckReplacements(line); })
       .Run(dry_run);
@@ -266,7 +269,8 @@ auto FileTestBase::DumpOutput() -> ErrorOr<Success> {
     return ErrorBuilder() << "Error updating " << test_name_ << ": "
                           << run_result.error();
   }
-  llvm::errs() << banner << context.stdout << banner << "= Exit with success: "
+  llvm::errs() << banner << context.actual_stdout << banner
+               << "= Exit with success: "
                << (context.run_result.success ? "true" : "false") << "\n"
                << banner;
   return Success();
@@ -297,18 +301,32 @@ auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
   CARBON_RETURN_IF_ERROR(
       DoArgReplacements(context.test_args, context.test_files));
 
+  // stdin needs to exist on-disk for compatibility. We'll use a pointer for it.
+  FILE* input_stream = nullptr;
+  auto erase_input_on_exit = llvm::make_scope_exit([&input_stream]() {
+    if (input_stream) {
+      // fclose should delete the tmpfile.
+      fclose(input_stream);
+      input_stream = nullptr;
+    }
+  });
+
   // Create the files in-memory.
   llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs =
       new llvm::vfs::InMemoryFileSystem;
   for (const auto& test_file : context.test_files) {
-    if (!fs->addFile(test_file.filename, /*ModificationTime=*/0,
-                     llvm::MemoryBuffer::getMemBuffer(
-                         test_file.content, test_file.filename,
-                         /*RequiresNullTerminator=*/false))) {
+    if (test_file.filename == StdinFilename) {
+      input_stream = tmpfile();
+      fwrite(test_file.content.c_str(), sizeof(char), test_file.content.size(),
+             input_stream);
+      rewind(input_stream);
+    } else if (!fs->addFile(test_file.filename, /*ModificationTime=*/0,
+                            llvm::MemoryBuffer::getMemBuffer(
+                                test_file.content, test_file.filename,
+                                /*RequiresNullTerminator=*/false))) {
       return ErrorBuilder() << "File is repeated: " << test_file.filename;
     }
   }
-
   // Convert the arguments to StringRef and const char* to match the
   // expectations of PrettyStackTraceProgram and Run.
   llvm::SmallVector<llvm::StringRef> test_args_ref;
@@ -340,20 +358,21 @@ auto FileTestBase::ProcessTestFileAndRun(TestContext& context)
     if (context.capture_console_output) {
       // No need to flush stderr.
       llvm::outs().flush();
-      context.stdout += GetCapturedStdout();
-      context.stderr += GetCapturedStderr();
+      context.actual_stdout += GetCapturedStdout();
+      context.actual_stderr += GetCapturedStderr();
     }
   });
 
   // Prepare string streams to capture output. In order to address casting
   // constraints, we split calls to Run as a ternary based on whether we want to
   // capture output.
-  llvm::raw_svector_ostream stdout(context.stdout);
-  llvm::raw_svector_ostream stderr(context.stderr);
+  llvm::raw_svector_ostream output_stream(context.actual_stdout);
+  llvm::raw_svector_ostream error_stream(context.actual_stderr);
   CARBON_ASSIGN_OR_RETURN(
       context.run_result,
-      context.dump_output ? Run(test_args_ref, fs, llvm::outs(), llvm::errs())
-                          : Run(test_args_ref, fs, stdout, stderr));
+      context.dump_output
+          ? Run(test_args_ref, fs, input_stream, llvm::outs(), llvm::errs())
+          : Run(test_args_ref, fs, input_stream, output_stream, error_stream));
 
   return Success();
 }
@@ -380,10 +399,11 @@ auto FileTestBase::DoArgReplacements(
         it = test_args.erase(it);
         for (const auto& file : test_files) {
           const std::string& filename = file.filename;
-          if (!filename.ends_with(".h")) {
-            it = test_args.insert(it, filename);
-            ++it;
+          if (filename == StdinFilename || filename.ends_with(".h")) {
+            continue;
           }
+          it = test_args.insert(it, filename);
+          ++it;
         }
         // Back up once because the for loop will advance.
         --it;
@@ -546,7 +566,6 @@ static auto AddTestFile(llvm::StringRef filename, std::string* content,
                         llvm::SmallVector<FileTestBase::TestFile>* test_files)
     -> ErrorOr<Success> {
   CARBON_RETURN_IF_ERROR(ReplaceContentKeywords(filename, content));
-
   test_files->push_back(
       {.filename = filename.str(), .content = std::move(*content)});
   content->clear();

testing/file_test/file_test_base.h

@@ -68,18 +68,23 @@ class FileTestBase : public testing::Test {
   explicit FileTestBase(std::mutex* output_mutex, llvm::StringRef test_name)
       : output_mutex_(output_mutex), test_name_(test_name) {}
 
-  // Implemented by children to run the test. For example, TestBody validates
-  // stdout and stderr. Children should use fs for file content, and may add
-  // more files.
+  // Implemented by children to run the test. The framework will validate the
+  // content written to `output_stream` and `error_stream`. Children should use
+  // `fs` for file content, and may add more files.
+  //
+  // If there is a split test file named "STDIN", then its contents will be
+  // provided at `input_stream` instead of `fs`. Otherwise, `input_stream` will
+  // be null.
   //
   // Any test expectations should be called from ValidateRun, not Run.
   //
   // The return value should be an error if there was an abnormal error, and
   // RunResult otherwise.
   virtual auto Run(const llvm::SmallVector<llvm::StringRef>& test_args,
                    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem>& fs,
-                   llvm::raw_pwrite_stream& stdout,
-                   llvm::raw_pwrite_stream& stderr) -> ErrorOr<RunResult> = 0;
+                   FILE* input_stream, llvm::raw_pwrite_stream& output_stream,
+                   llvm::raw_pwrite_stream& error_stream)
+      -> ErrorOr<RunResult> = 0;
 
   // Implemented by children to do post-Run test expectations. Only called when
   // testing. Does not need to be provided if only CHECK test expectations are
@@ -174,8 +179,8 @@ class FileTestBase : public testing::Test {
     llvm::SmallVector<testing::Matcher<std::string>> expected_stderr;
 
     // stdout and stderr from Run. 16 is arbitrary but a required value.
-    llvm::SmallString<16> stdout;
-    llvm::SmallString<16> stderr;
+    llvm::SmallString<16> actual_stdout;
+    llvm::SmallString<16> actual_stderr;
 
     RunResult run_result = {.success = false};
   };