integration_tests: bootstrap all the way to being able to run a basic…

… sine test

Cargo.toml

@@ -53,6 +53,7 @@ regex = "1.10.3"
 rubato = "0.14.1"
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"
+serde_yaml = "0.9.32"
 sharded-slab = "0.1.4"
 smallvec = { version = "1.10.0", features = ["write"] }
 spin = "0.9.8"

crates/integration_tests/Cargo.toml

@@ -22,4 +22,5 @@ paste.workspace = true
 regex.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+serde_yaml.workspace = true
 synthizer.workspace = true

crates/integration_tests/src/cli_args.rs

@@ -17,6 +17,11 @@ pub enum Command {
     /// List all tests.
     List(ListArgs),
 
+    /// Given the name of a test, pretty print the response.json or response-panic.json
+    ///
+    /// Figures out whichever is appropriate, and then shows that.
+    ViewResponse(ViewResponseArgs),
+
     /// Private command used as an entrypoint to subprocesses.
     SubprocessEntryPoint(SubprocessArgs),
 }
@@ -47,3 +52,8 @@ pub struct ListArgs {
     #[command(flatten)]
     pub filter: FilterArgs,
 }
+
+#[derive(Debug, Parser)]
+pub struct ViewResponseArgs {
+    pub test_name: String,
+}

crates/integration_tests/src/commands/mod.rs

@@ -1,13 +1,15 @@
 mod list;
 mod run;
 mod subprocess_entry_point;
+mod view_response;
 
 use crate::cli_args;
 
 /// Figure out what command to run, then run it.
 pub fn dispatch_command(args: cli_args::CliArgs) {
     match &args.command {
         cli_args::Command::List(l) => list::list(&args, l),
+        cli_args::Command::ViewResponse(r) => view_response::view_response(&args, r),
         cli_args::Command::Run(r) => run::run(&args, r),
         cli_args::Command::SubprocessEntryPoint(sp) => {
             subprocess_entry_point::subprocess_entry_point(&args, sp)

crates/integration_tests/src/commands/view_response.rs

@@ -0,0 +1,48 @@
+use crate::cli_args::*;
+
+pub fn view_response(_top_args: &CliArgs, cmd_args: &ViewResponseArgs) {
+    let env = crate::environment::get_env();
+
+    if crate::registry::get_test_by_name(&cmd_args.test_name).is_none() {
+        panic!("{} is not a valid, registered test", cmd_args.test_name);
+    };
+
+    let artifacts_dir = env.artifacts_dir_for(&cmd_args.test_name);
+    if !artifacts_dir.exists() {
+        panic!(
+            "{}: no artifacts directory found. Tried {}. This probably means the test passed.",
+            cmd_args.test_name,
+            artifacts_dir.display()
+        );
+    }
+
+    let possibilities = [
+        env.panic_response_file_for(&cmd_args.test_name),
+        env.good_response_file_for(&cmd_args.test_name),
+    ];
+
+    let mut response: Option<String> = None;
+    for p in possibilities {
+        match std::fs::read(&p) {
+            Ok(r) => {
+                response = Some(
+                    String::from_utf8(r).expect("This is a JSON file and should always be UTF-8"),
+                );
+            }
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
+            Err(e) => {
+                panic!("While trying to open path {}: {}", p.display(), e);
+            }
+        }
+    }
+
+    let response = response.expect(
+        "Unable to find a response file in the artifacts directory. Manual examination is required",
+    );
+
+    // Converting to yaml gets us nice pretty printing of multiline strings.
+    let json: serde_json::Value = serde_json::from_str(&response).unwrap();
+    let yaml = serde_yaml::to_string(&json).unwrap();
+    println!("Response for {}", cmd_args.test_name);
+    println!("{yaml}");
+}

crates/integration_tests/src/context.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use std::time::Duration;
 
 use synthizer as syz;
 
@@ -8,15 +9,56 @@ use crate::validators::Validator;
 pub struct TestContext {
     pub test_name: String,
     pub channel_format: syz::ChannelFormat,
+    pub server: syz::Server,
     pub validators: Vec<Box<dyn Validator>>,
+
+    /// Last chunk of synthesized audio, if any.
+    ///
+    /// On failure to advance, set to the empty vec.
+    pub synthesized_audio: Vec<f32>,
+}
+
+/// Trait representing various units of time.
+pub trait Advanceable {
+    fn get_frame_count(&self) -> usize;
+}
+
+/// Advance by some number of frames.
+pub struct Frames(pub usize);
+
+/// Advance by some number of blocks.
+pub struct Blocks(pub usize);
+
+impl Advanceable for Frames {
+    fn get_frame_count(&self) -> usize {
+        self.0
+    }
+}
+
+impl Advanceable for Blocks {
+    fn get_frame_count(&self) -> usize {
+        self.0.checked_mul(syz::BLOCK_SIZE).expect(
+            "Attempt to advance by more than the number of frames that could ever fit into memory",
+        )
+    }
 }
 
+impl Advanceable for Duration {
+    fn get_frame_count(&self) -> usize {
+        let secs = self.as_secs_f64();
+        let frames = secs * (syz::SR as f64);
+        (frames.ceil() as u64).try_into().expect("Unable to advance by this many seconds because that would require more frames than can fit into the memory of this machine")
+    }
+}
 impl TestContext {
     pub fn from_config(test_name: &str, config: TestConfig) -> Result<TestContext> {
         let mut ret = Self {
             channel_format: syz::ChannelFormat::Stereo,
+            server: syz::Server::new_inline()
+                .expect("Must be able to create a Synthizer server for test startup"),
             test_name: test_name.to_string(),
             validators: vec![],
+            synthesized_audio: vec![0.0; syz::BLOCK_SIZE * 2],
         };
 
         let validators = config
@@ -29,14 +71,55 @@ impl TestContext {
         Ok(ret)
     }
 
+    /// Run the specified closure over all validators in such a way as to allow it to itself get a context.
+    ///
+    /// Deals with the limitation of Rust that we cannot do field splitrting over the whole call grapha by taking the
+    /// validators out of the context, then putting them back.
+    fn validators_foreach(&mut self, mut callback: impl FnMut(&TestContext, &mut dyn Validator)) {
+        let mut vals = std::mem::take(&mut self.validators);
+        for v in vals.iter_mut() {
+            callback(self, &mut **v);
+        }
+        self.validators = vals;
+    }
+
     /// Get the outcomes of all validators.
     pub fn finalize_validators(&mut self) -> Vec<Result<(), crate::validators::ValidatorFailure>> {
-        // We need to be able to let the validators see the context, so take the list of validators out, then put it
-        // back.
-        let mut validators = std::mem::take(&mut self.validators);
-
-        let ret = validators.iter_mut().map(|x| x.finalize(self)).collect();
-        self.validators = validators;
+        let mut ret = vec![];
+        self.validators_foreach(|ctx, v| {
+            ret.push(v.finalize(ctx));
+        });
         ret
     }
+
+    #[track_caller]
+    fn advance_by_frames(&mut self, frame_count: usize) -> Result<()> {
+        let mut synthesized_audio = std::mem::take(&mut self.synthesized_audio);
+        synthesized_audio.resize(
+            frame_count * self.channel_format.get_channel_count().get(),
+            0.0,
+        );
+        // We want to test that Synthizer zeros this buffer, so we can fill it with NaN and then tests freak out if this
+        // is not the case.
+        synthesized_audio.fill(f32::NAN);
+
+        self.server.synthesize_stereo(&mut synthesized_audio[..])?;
+
+        // Must be here, otherwise track_caller doesn't work through the closure.
+        let loc = std::panic::Location::caller();
+        self.validators_foreach(|ctx, v| {
+            v.validate_batched(ctx, loc, &synthesized_audio[..]);
+        });
+        Ok(())
+    }
+
+    /// Advance this simulation.  Time may be:
+    ///
+    /// - A [Duration]. This is converted to samples and rounded up to the next sample.
+    /// - [Frames]: E.g. `Frames(2)`.  Advance by the specified number of frames.
+    /// - [Blocks]: e.g. `Blocks(5)`.  Advance by the specified number of blocks.
+    #[track_caller]
+    pub fn advance<T: Advanceable>(&mut self, time: T) -> Result<()> {
+        self.advance_by_frames(time.get_frame_count())
+    }
 }

crates/integration_tests/src/environment.rs

@@ -60,4 +60,8 @@ impl<T: AsRef<Path>> Environment<T> {
     pub fn panic_response_file_for(&self, test_name: &str) -> PathBuf {
         self.artifacts_dir_for(test_name).join(RESPONSE_PANIC_FILE)
     }
+
+    pub fn good_response_file_for(&self, test_name: &str) -> PathBuf {
+        self.artifacts_dir_for(test_name).join(RESPONSE_GOOD_FILE)
+    }
 }

crates/integration_tests/src/process_coordination/protocol.rs

@@ -32,7 +32,7 @@ pub struct SubprocessResponse {
 }
 
 /// The outcome of a test.
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, derive_more::IsVariant)]
 pub enum TestOutcome {
     /// Test passed this time.  If there are further runs, cancel them.
     Passed,

crates/integration_tests/src/registry.rs

@@ -65,6 +65,9 @@ fn build_test_cache() -> Vec<&'static TestRegistryEntry> {
     ret
 }
 
+pub fn get_test_by_name(test_name: &str) -> Option<&'static TestRegistryEntry> {
+    get_tests().find(|x| x.name() == test_name)
+}
 impl TestRegistryEntry {
     pub fn name(&self) -> &str {
         self.name

crates/integration_tests/src/reporter.rs

@@ -43,23 +43,30 @@ fn report_test_fallible(
     write!(dest, "{test_name} ")?;
 
     match outcome {
-        proto::TestOutcome::Passed => write!(dest, "passed"),
+        proto::TestOutcome::Passed => write!(dest, "passed")?,
         proto::TestOutcome::Panicked(p) => {
             writeln!(dest, "panicked")?;
-            report_panic(&mut indented(&mut dest).ind(2), test_name, p)
+            report_panic(&mut indented(&mut dest).with_str("  "), test_name, p)?;
         }
         proto::TestOutcome::RunnerFailed(r) => {
             writeln!(dest, "Runner Failed")?;
-            report_runner_failed(&mut indented(&mut dest).ind(2), r)
+            report_runner_failed(&mut indented(&mut dest).with_str("  "), r)?;
         }
         proto::TestOutcome::ValidatorsFailed(v) => {
             writeln!(dest, "Validators failed")?;
-            report_validators_failed(&mut indented(&mut dest).ind(2), test_config, v)
+            report_validators_failed(&mut indented(&mut dest).with_str("  "), test_config, v)?;
         }
         proto::TestOutcome::Indeterminate => {
-            write!(dest, "Ended with an indeterminate result")
+            writeln!(dest, "Ended with an indeterminate result")?;
         }
     }
+
+    if !outcome.is_passed() {
+        // Mind the double spaces at the beginning of this string.
+        writeln!(dest, "  More information may be available. Try cargo run --bin synthizer_integration_tests -- view-response {test_name}")?;
+    }
+
+    Ok(())
 }
 
 fn report_panic(dest: &mut dyn Write, test_name: &str, info: &proto::PanicOutcome) -> Result {
@@ -86,8 +93,8 @@ fn report_validators_failed(
     for v in info.entries.iter() {
         let mut ind_fmt = indented(&mut dest);
         let tag = test_config.validators[v.index].get_tag();
-        write!(&mut ind_fmt, "Validator {} (a {tag}): ", v.index)?;
-        writeln!(ind_fmt.ind(2), "{}", v.payload)?;
+        writeln!(&mut ind_fmt, "Validator {} (a {tag}): ", v.index)?;
+        writeln!(ind_fmt.with_str("  "), "{}", v.payload)?;
     }
     Ok(())
 }

crates/integration_tests/src/test_config.rs

@@ -11,6 +11,7 @@ pub struct TestConfig {
     ///
     /// This is used basically only for the self test where we have a test that does nothing to make sure the framework
     /// works.  Other tests shouldn't have it, save while debugging weirdness, since the framework writes large files.
+    #[builder(default)]
     pub keep_artifacts_on_success: bool,
 }
 

crates/integration_tests/src/test_runner.rs

@@ -80,9 +80,7 @@ pub fn run_single_test_in_parent(name: &str) -> Result<protocol::SubprocessRespo
     }
 
     let outcome = crate::process_coordination::parent_process::parent_process(name)?;
-    if matches!(&outcome.outcome, protocol::TestOutcome::Passed)
-        && !config.keep_artifacts_on_success
-    {
+    if outcome.outcome.is_passed() && !config.keep_artifacts_on_success {
         std::fs::remove_dir_all(&artifacts_directory).context(format!("While trying to clean up the artifacts directory after a successful test name={name}: {}", artifacts_directory.display()))?;
     }
 
@@ -93,11 +91,17 @@ pub fn run_single_test_in_parent(name: &str) -> Result<protocol::SubprocessRespo
 ///
 /// This function never returns, and exits the process with the appropriate error code.
 pub fn run_tests(filter: &crate::cli_args::FilterArgs) {
+    let mut all_passed = true;
+
     for test in crate::test_filtering::get_tests_filtered(filter) {
         let name = test.name();
         let res = run_single_test_in_parent(test.name())
             .expect("Running tests themselves should always work unless the harness is bugged or the environment is bad");
         let reported = crate::reporter::report_test(name, &(test.config_fn)(), &res.outcome);
+        all_passed &= matches!(res.outcome, protocol::TestOutcome::Passed);
         eprintln!("{reported}");
     }
+
+    let exit_code = (!all_passed) as _;
+    std::process::exit(exit_code);
 }