diff --git a/Cargo.lock b/Cargo.lock index e0f3537..8cebe67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,6 +791,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" version = "1.9.3" @@ -1898,6 +1904,7 @@ dependencies = [ "derive_builder", "derive_more", "globset", + "indenter", "inventory", "itertools", "lazy_static", diff --git a/Cargo.toml b/Cargo.toml index 6f1f1d4..ea060f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ eye_dropper = { path = "crates/eye_dropper" } globset = "0.4.14" hound = "3.5.1" im = "15.1.0" +indenter = { version = "0.3.3", features = ["std"] } inventory = "0.3.15" itertools = "0.10.5" lazy_static = "1.4.0" diff --git a/crates/integration_tests/Cargo.toml b/crates/integration_tests/Cargo.toml index 6cd4278..0cd1772 100644 --- a/crates/integration_tests/Cargo.toml +++ b/crates/integration_tests/Cargo.toml @@ -12,6 +12,7 @@ clap.workspace = true derive_builder.workspace = true derive_more.workspace = true globset.workspace = true +indenter.workspace = true inventory.workspace = true itertools.workspace = true lazy_static.workspace = true diff --git a/crates/integration_tests/src/environment.rs b/crates/integration_tests/src/environment.rs index 85b461a..976ccb7 100644 --- a/crates/integration_tests/src/environment.rs +++ b/crates/integration_tests/src/environment.rs @@ -51,3 +51,13 @@ pub const RESPONSE_GOOD_FILE: &str = "response.json"; /// File to which a response is written if a test panics. pub const RESPONSE_PANIC_FILE: &str = "response-panic.json"; + +impl> Environment { + pub fn artifacts_dir_for(&self, test_name: &str) -> PathBuf { + self.temp_artifacts_dir.as_ref().join(test_name) + } + + pub fn panic_response_file_for(&self, test_name: &str) -> PathBuf { + self.artifacts_dir_for(test_name).join(RESPONSE_PANIC_FILE) + } +} diff --git a/crates/integration_tests/src/main.rs b/crates/integration_tests/src/main.rs index 1b70229..728a1c9 100644 --- a/crates/integration_tests/src/main.rs +++ b/crates/integration_tests/src/main.rs @@ -58,6 +58,7 @@ mod context; mod environment; mod process_coordination; mod registry; +mod reporter; mod test_config; mod test_filtering; mod test_runner; diff --git a/crates/integration_tests/src/process_coordination/protocol.rs b/crates/integration_tests/src/process_coordination/protocol.rs index 3dfb3a5..739ea2c 100644 --- a/crates/integration_tests/src/process_coordination/protocol.rs +++ b/crates/integration_tests/src/process_coordination/protocol.rs @@ -46,7 +46,7 @@ pub enum TestOutcome { Indeterminate, /// The test failed because some validator did. - ValidatorsFailed(ValidatorsFailed), + ValidatorsFailed(ValidatorsFailedResponse), /// The process panicked. Could be an assert or an actual problem. Panicked(PanicOutcome), @@ -64,7 +64,7 @@ pub struct FailedValidatorEntry { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ValidatorsFailed { +pub struct ValidatorsFailedResponse { pub entries: Vec, } diff --git a/crates/integration_tests/src/reporter.rs b/crates/integration_tests/src/reporter.rs new file mode 100644 index 0000000..8f227d8 --- /dev/null +++ b/crates/integration_tests/src/reporter.rs @@ -0,0 +1,93 @@ +//! Infrastructure to report the results of running a test. +//! +//! This is called from [crate::test_runner] to display the outputs of tests. +use std::fmt::{Result, Write}; + +use indenter::indented; + +use crate::environment::get_env; +use crate::process_coordination::protocol as proto; + +// Implementation: we have a root entrypoint at `report_test`, then we use indenter to bring everything together. +// Formatting here is to strings and so cannot fail, but unwrap is annoying so we put that behind a function and unwrap +// once at the top. +// +// The output string does not contain a newline, and this is handled by stripping at the top. This lets us use writeln +// everywhere. + +/// Report the outcome of a test. +/// +/// Returns a string without a trailing newline. +pub fn report_test( + test_name: &str, + test_config: &crate::test_config::TestConfig, + outcome: &proto::TestOutcome, +) -> String { + let mut dest = String::new(); + report_test_fallible(&mut dest, test_name, test_config, outcome) + .expect("This is formatting to strings and should never fail"); + + // it's really hard to get newlines right, so we strip at the top. + let Some(stripped) = dest.strip_suffix('\n') else { + return dest; + }; + stripped.to_string() +} + +fn report_test_fallible( + mut dest: &mut dyn Write, + test_name: &str, + test_config: &crate::test_config::TestConfig, + outcome: &proto::TestOutcome, +) -> Result { + write!(dest, "{test_name} ")?; + + match outcome { + proto::TestOutcome::Passed => write!(dest, "passed"), + proto::TestOutcome::Panicked(p) => { + writeln!(dest, "panicked")?; + report_panic(&mut indented(&mut dest).ind(2), test_name, p) + } + proto::TestOutcome::RunnerFailed(r) => { + writeln!(dest, "Runner Failed")?; + report_runner_failed(&mut indented(&mut dest).ind(2), r) + } + proto::TestOutcome::ValidatorsFailed(v) => { + writeln!(dest, "Validators failed")?; + report_validators_failed(&mut indented(&mut dest).ind(2), test_config, v) + } + proto::TestOutcome::Indeterminate => { + write!(dest, "Ended with an indeterminate result") + } + } +} + +fn report_panic(dest: &mut dyn Write, test_name: &str, info: &proto::PanicOutcome) -> Result { + let panic_resp = get_env().panic_response_file_for(test_name); + let pan_info = &info.panic_info; + let loc = info.location.as_deref().unwrap_or("UNAVAILABLE"); + writeln!(dest, "{pan_info}")?; + writeln!(dest, "Location: {loc}")?; + writeln!(dest, "NOTE: more info in {}", panic_resp.display())?; + Ok(()) +} + +fn report_runner_failed(dest: &mut dyn Write, info: &proto::RunnerFailedResponse) -> Result { + writeln!(dest, "Reason: {}", info.reason) +} + +fn report_validators_failed( + mut dest: &mut dyn Write, + test_config: &crate::test_config::TestConfig, + info: &proto::ValidatorsFailedResponse, +) -> Result { + writeln!(dest, "{} validators have failed", info.entries.len())?; + + for v in info.entries.iter() { + let mut ind_fmt = indented(&mut dest); + let tag = test_config.validators[v.index].get_tag(); + write!(&mut ind_fmt, "Validator {} (a {tag}): ", v.index)?; + writeln!(ind_fmt.ind(2), "{}", v.payload)?; + } + Ok(()) +} diff --git a/crates/integration_tests/src/test_runner.rs b/crates/integration_tests/src/test_runner.rs index ee31fa1..f443419 100644 --- a/crates/integration_tests/src/test_runner.rs +++ b/crates/integration_tests/src/test_runner.rs @@ -38,7 +38,7 @@ pub fn run_single_test_in_subprocess(name: &str) -> Result Result, _context: &TestContext) -> Box { Box::new(FunctionValidator::KeepGoing(*self)) } + + fn get_tag(&self) -> &str { + "Closure" + } } diff --git a/crates/integration_tests/src/validators/mod.rs b/crates/integration_tests/src/validators/mod.rs index 74f3529..8b85419 100644 --- a/crates/integration_tests/src/validators/mod.rs +++ b/crates/integration_tests/src/validators/mod.rs @@ -13,8 +13,9 @@ pub use range::*; /// Reasons a validator may fail. /// /// Some validators are able to provide more semantic information than a string. This enum allows capturing that. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, derive_more::Display, Serialize, Deserialize)] pub enum ValidatorFailure { + #[display(fmt = "{}", _0)] SimpleMessage(String), } @@ -68,4 +69,9 @@ pub trait Validator: Send + Sync + 'static { /// when the exact sequence is known to some tolerance. pub trait IntoValidator: 'static { fn build_validator(self: Box, context: &TestContext) -> Box; + + /// The tag of a validator is the name, e.g. "golden", "closure". + /// + /// Used for printing test results. + fn get_tag(&self) -> &str; } diff --git a/crates/integration_tests/src/validators/range.rs b/crates/integration_tests/src/validators/range.rs index ab43450..42be80a 100644 --- a/crates/integration_tests/src/validators/range.rs +++ b/crates/integration_tests/src/validators/range.rs @@ -75,4 +75,8 @@ impl IntoValidator for RangeValidator { current_index: 0, }) } + + fn get_tag(&self) -> &str { + "RangeValidator" + } }