Lots of reporting stuff

Cargo.toml

@@ -27,6 +27,7 @@ eye_dropper = { path = "crates/eye_dropper" }
 globset = "0.4.14"
 hound = "3.5.1"
 im = "15.1.0"
+indenter = { version = "0.3.3", features = ["std"] }
 inventory = "0.3.15"
 itertools = "0.10.5"
 lazy_static = "1.4.0"

crates/integration_tests/Cargo.toml

@@ -12,6 +12,7 @@ clap.workspace = true
 derive_builder.workspace = true
 derive_more.workspace = true
 globset.workspace = true
+indenter.workspace = true
 inventory.workspace = true
 itertools.workspace = true
 lazy_static.workspace = true

crates/integration_tests/src/environment.rs

@@ -51,3 +51,13 @@ pub const RESPONSE_GOOD_FILE: &str = "response.json";
 
 /// File to which a response is written if a test panics.
 pub const RESPONSE_PANIC_FILE: &str = "response-panic.json";
+
+impl<T: AsRef<Path>> Environment<T> {
+    pub fn artifacts_dir_for(&self, test_name: &str) -> PathBuf {
+        self.temp_artifacts_dir.as_ref().join(test_name)
+    }
+
+    pub fn panic_response_file_for(&self, test_name: &str) -> PathBuf {
+        self.artifacts_dir_for(test_name).join(RESPONSE_PANIC_FILE)
+    }
+}

crates/integration_tests/src/main.rs

@@ -58,6 +58,7 @@ mod context;
 mod environment;
 mod process_coordination;
 mod registry;
+mod reporter;
 mod test_config;
 mod test_filtering;
 mod test_runner;

crates/integration_tests/src/process_coordination/protocol.rs

@@ -46,7 +46,7 @@ pub enum TestOutcome {
     Indeterminate,
 
     /// The test failed because some validator did.
-    ValidatorsFailed(ValidatorsFailed),
+    ValidatorsFailed(ValidatorsFailedResponse),
 
     /// The process panicked.  Could be an assert or an actual problem.
     Panicked(PanicOutcome),
@@ -64,7 +64,7 @@ pub struct FailedValidatorEntry {
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct ValidatorsFailed {
+pub struct ValidatorsFailedResponse {
     pub entries: Vec<FailedValidatorEntry>,
 }
 

crates/integration_tests/src/reporter.rs

@@ -0,0 +1,93 @@
+//! Infrastructure to report the results of running a test.
+//!
+//! This is called from [crate::test_runner] to display the outputs of tests.
+use std::fmt::{Result, Write};
+
+use indenter::indented;
+
+use crate::environment::get_env;
+use crate::process_coordination::protocol as proto;
+
+// Implementation: we have a root entrypoint at `report_test`, then we use indenter to bring everything together.
+// Formatting here is to strings and so cannot fail, but unwrap is annoying so we put that behind a function and unwrap
+// once at the top.
+//
+// The output string does not contain a newline, and this is handled by stripping at the top.  This lets us use writeln
+// everywhere.
+
+/// Report the outcome of a test.
+///
+/// Returns a string without a trailing newline.
+pub fn report_test(
+    test_name: &str,
+    test_config: &crate::test_config::TestConfig,
+    outcome: &proto::TestOutcome,
+) -> String {
+    let mut dest = String::new();
+    report_test_fallible(&mut dest, test_name, test_config, outcome)
+        .expect("This is formatting to strings and should never fail");
+
+    // it's really hard to get newlines right, so we strip at the top.
+    let Some(stripped) = dest.strip_suffix('\n') else {
+        return dest;
+    };
+    stripped.to_string()
+}
+
+fn report_test_fallible(
+    mut dest: &mut dyn Write,
+    test_name: &str,
+    test_config: &crate::test_config::TestConfig,
+    outcome: &proto::TestOutcome,
+) -> Result {
+    write!(dest, "{test_name} ")?;
+
+    match outcome {
+        proto::TestOutcome::Passed => write!(dest, "passed"),
+        proto::TestOutcome::Panicked(p) => {
+            writeln!(dest, "panicked")?;
+            report_panic(&mut indented(&mut dest).ind(2), test_name, p)
+        }
+        proto::TestOutcome::RunnerFailed(r) => {
+            writeln!(dest, "Runner Failed")?;
+            report_runner_failed(&mut indented(&mut dest).ind(2), r)
+        }
+        proto::TestOutcome::ValidatorsFailed(v) => {
+            writeln!(dest, "Validators failed")?;
+            report_validators_failed(&mut indented(&mut dest).ind(2), test_config, v)
+        }
+        proto::TestOutcome::Indeterminate => {
+            write!(dest, "Ended with an indeterminate result")
+        }
+    }
+}
+
+fn report_panic(dest: &mut dyn Write, test_name: &str, info: &proto::PanicOutcome) -> Result {
+    let panic_resp = get_env().panic_response_file_for(test_name);
+    let pan_info = &info.panic_info;
+    let loc = info.location.as_deref().unwrap_or("UNAVAILABLE");
+    writeln!(dest, "{pan_info}")?;
+    writeln!(dest, "Location: {loc}")?;
+    writeln!(dest, "NOTE: more info in {}", panic_resp.display())?;
+    Ok(())
+}
+
+fn report_runner_failed(dest: &mut dyn Write, info: &proto::RunnerFailedResponse) -> Result {
+    writeln!(dest, "Reason: {}", info.reason)
+}
+
+fn report_validators_failed(
+    mut dest: &mut dyn Write,
+    test_config: &crate::test_config::TestConfig,
+    info: &proto::ValidatorsFailedResponse,
+) -> Result {
+    writeln!(dest, "{} validators have failed", info.entries.len())?;
+
+    for v in info.entries.iter() {
+        let mut ind_fmt = indented(&mut dest);
+        let tag = test_config.validators[v.index].get_tag();
+        write!(&mut ind_fmt, "Validator {} (a {tag}): ", v.index)?;
+        writeln!(ind_fmt.ind(2), "{}", v.payload)?;
+    }
+    Ok(())
+}

crates/integration_tests/src/test_runner.rs

@@ -38,7 +38,7 @@ pub fn run_single_test_in_subprocess(name: &str) -> Result<protocol::SubprocessR
     // If any validator failed, tel the parent.
     if !validator_failures.is_empty() {
         return Ok(protocol::SubprocessResponse {
-            outcome: protocol::TestOutcome::ValidatorsFailed(protocol::ValidatorsFailed {
+            outcome: protocol::TestOutcome::ValidatorsFailed(protocol::ValidatorsFailedResponse {
                 entries: validator_failures,
             }),
         });
@@ -91,10 +91,13 @@ pub fn run_single_test_in_parent(name: &str) -> Result<protocol::SubprocessRespo
 
 /// Filter down the tests with the provided filter and run them all.  Then, report the outcome to stderr.
 ///
-/// On unrecoverable error, exits the process.
+/// This function never returns, and exits the process with the appropriate error code.
 pub fn run_tests(filter: &crate::cli_args::FilterArgs) {
     for test in crate::test_filtering::get_tests_filtered(filter) {
         let name = test.name();
-        eprintln!("{name}: {:?}", run_single_test_in_parent(test.name()));
+        let res = run_single_test_in_parent(test.name())
+            .expect("Running tests themselves should always work unless the harness is bugged or the environment is bad");
+        let reported = crate::reporter::report_test(name, &(test.config_fn)(), &res.outcome);
+        eprintln!("{reported}");
     }
 }

crates/integration_tests/src/validators/function.rs

@@ -42,4 +42,8 @@ where
     fn build_validator(self: Box<Self>, _context: &TestContext) -> Box<dyn super::Validator> {
         Box::new(FunctionValidator::KeepGoing(*self))
     }
+
+    fn get_tag(&self) -> &str {
+        "Closure"
+    }
 }

crates/integration_tests/src/validators/mod.rs

@@ -13,8 +13,9 @@ pub use range::*;
 /// Reasons a validator may fail.
 ///
 /// Some validators are able to provide more semantic information than a string. This enum allows capturing that.
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, derive_more::Display, Serialize, Deserialize)]
 pub enum ValidatorFailure {
+    #[display(fmt = "{}", _0)]
     SimpleMessage(String),
 }
 
@@ -68,4 +69,9 @@ pub trait Validator: Send + Sync + 'static {
 /// when the exact sequence is known to some tolerance.
 pub trait IntoValidator: 'static {
     fn build_validator(self: Box<Self>, context: &TestContext) -> Box<dyn Validator>;
+
+    /// The tag of a validator is the name, e.g. "golden", "closure".
+    ///
+    /// Used for printing test results.
+    fn get_tag(&self) -> &str;
 }

crates/integration_tests/src/validators/range.rs

@@ -75,4 +75,8 @@ impl IntoValidator for RangeValidator {
             current_index: 0,
         })
     }
+
+    fn get_tag(&self) -> &str {
+        "RangeValidator"
+    }
 }