diff --git a/.cspell.json b/.cspell.json index 9655369..4443641 100644 --- a/.cspell.json +++ b/.cspell.json @@ -38,6 +38,7 @@ "Chunker", "cmdline", "composability", + "cpus", "discoverability", "Discoverability", "editenv", diff --git a/crates/reportify/src/lib.rs b/crates/reportify/src/lib.rs index 44be1bc..fe5bf63 100644 --- a/crates/reportify/src/lib.rs +++ b/crates/reportify/src/lib.rs @@ -276,6 +276,7 @@ use std::{ any::Any, error::Error as StdError, fmt::{Debug, Display}, + future::Future, }; use backtrace::BacktraceImpl; @@ -327,7 +328,7 @@ impl Error for E { } } -trait AnyReport { +trait AnyReport: Send { fn error(&self) -> &dyn Error; fn meta(&self) -> &ReportMeta; diff --git a/crates/rugpi-bakery/src/bake/image.rs b/crates/rugpi-bakery/src/bake/image.rs index 2af7445..762c538 100644 --- a/crates/rugpi-bakery/src/bake/image.rs +++ b/crates/rugpi-bakery/src/bake/image.rs @@ -25,7 +25,7 @@ use crate::{ rpi_uboot::initialize_uboot, Target, }, project::images::{self, grub_efi_image_layout, pi_image_layout, ImageConfig, ImageLayout}, - utils::prelude::*, + utils::{caching::mtime, prelude::*}, BakeryResult, }; @@ -37,6 +37,15 @@ pub fn make_image(config: &ImageConfig, src: &Path, image: &Path) -> BakeryResul fs::create_dir_all(parent).ok(); } + if image.exists() { + let src_mtime = mtime(src).whatever("unable to get image mtime")?; + let image_mtime = mtime(image).whatever("unable to get image mtime")?; + if src_mtime <= image_mtime { + info!("Image is newer than sources."); + return Ok(()); + } + } + // Initialize system root directory from provided TAR file. info!("Extracting layer."); run!(["tar", "-xf", src, "-C", &bundle_dir]).whatever("unable to extract layer")?; diff --git a/crates/rugpi-bakery/src/main.rs b/crates/rugpi-bakery/src/main.rs index 88a60f0..f374131 100644 --- a/crates/rugpi-bakery/src/main.rs +++ b/crates/rugpi-bakery/src/main.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, convert::Infallible, - ffi::{CStr, CString}, + ffi::{CStr, CString, OsStr}, fs, path::{Path, PathBuf}, }; @@ -15,6 +15,7 @@ use project::{ use reportify::{bail, Report, ResultExt}; use rugpi_common::fsutils::copy_recursive; use serde::Deserialize; +use test::RugpiTestError; pub mod bake; pub mod project; @@ -93,7 +94,7 @@ pub enum BakeCommand { /// The `test` command. #[derive(Debug, Parser)] pub struct TestCommand { - case: PathBuf, + workflows: Vec, } /// The `bake` command. @@ -233,11 +234,38 @@ fn main() -> BakeryResult<()> { } } Command::Test(test_command) => { + let project = load_project(&args)?; tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .unwrap() - .block_on(test::main(&test_command.case)) + .block_on(async move { + let mut workflows = Vec::new(); + if test_command.workflows.is_empty() { + let mut read_dir = tokio::fs::read_dir(project.dir.join("tests")) + .await + .whatever("unable to scan for test workflows")?; + while let Some(entry) = read_dir + .next_entry() + .await + .whatever("unable to read entry")? + { + let path = entry.path(); + if path.extension() == Some(OsStr::new("toml")) { + workflows.push(path); + } + } + } else { + for name in &test_command.workflows { + workflows + .push(project.dir.join("tests").join(name).with_extension("toml")); + } + }; + for workflow in workflows { + test::main(&project, &workflow).await?; + } + >>::Ok(()) + }) .whatever("unable to run test")?; } } diff --git a/crates/rugpi-bakery/src/project/mod.rs b/crates/rugpi-bakery/src/project/mod.rs index 1f7040c..33f9780 100644 --- a/crates/rugpi-bakery/src/project/mod.rs +++ b/crates/rugpi-bakery/src/project/mod.rs @@ -1,15 +1,14 @@ //! In-memory representation of Rugpi Bakery projects. use std::{ - cell::OnceCell, path::{Path, PathBuf}, - sync::Arc, + sync::{Arc, OnceLock}, }; use reportify::ResultExt; use self::{config::BakeryConfig, library::Library, repositories::ProjectRepositories}; -use crate::{utils::prelude::*, BakeryResult}; +use crate::BakeryResult; pub mod config; pub mod images; @@ -19,7 +18,7 @@ pub mod recipes; pub mod repositories; /// A project. -#[derive(Debug)] +#[derive(Debug, Clone)] #[non_exhaustive] pub struct Project { /// The configuration of the project. @@ -33,30 +32,37 @@ pub struct Project { impl Project { /// The repositories of the project. pub fn repositories(&self) -> BakeryResult<&Arc> { - self.lazy - .repositories - .try_get_or_init(|| ProjectRepositories::load(self).map(Arc::new)) - .whatever("loading repositories") + if let Some(repositories) = self.lazy.repositories.get() { + return Ok(repositories); + } + let repositories = ProjectRepositories::load(self) + .map(Arc::new) + .whatever("loading repositories")?; + let _ = self.lazy.repositories.set(repositories); + Ok(self.lazy.repositories.get().unwrap()) } /// The library of the project. pub fn library(&self) -> BakeryResult<&Arc> { - self.lazy.library.try_get_or_init(|| { - let repositories = self.repositories()?.clone(); - Library::load(repositories) - .map(Arc::new) - .whatever("loading library") - }) + if let Some(library) = self.lazy.library.get() { + return Ok(library); + } + let repositories = self.repositories()?.clone(); + let library = Library::load(repositories) + .map(Arc::new) + .whatever("loading library")?; + let _ = self.lazy.library.set(library); + Ok(self.lazy.library.get().unwrap()) } } /// Lazily initialized fields of [`Project`]. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] struct ProjectLazy { /// The repositories of the project. - repositories: OnceCell>, + repositories: OnceLock>, /// The library of the project. - library: OnceCell>, + library: OnceLock>, } /// Project loader. diff --git a/crates/rugpi-bakery/src/test/mod.rs b/crates/rugpi-bakery/src/test/mod.rs index f1025ca..b01cdb7 100644 --- a/crates/rugpi-bakery/src/test/mod.rs +++ b/crates/rugpi-bakery/src/test/mod.rs @@ -1,12 +1,14 @@ use std::{path::Path, time::Duration}; -use case::{TestCase, TestStep}; use reportify::{bail, Report, ResultExt}; use rugpi_cli::info; -use tokio::fs; +use tokio::{fs, task::spawn_blocking}; +use workflow::{TestStep, TestWorkflow}; + +use crate::{bake, project::Project}; -pub mod case; pub mod qemu; +pub mod workflow; reportify::new_whatever_type! { RugpiTestError @@ -14,48 +16,62 @@ reportify::new_whatever_type! { pub type RugpiTestResult = Result>; -pub async fn main(case: &Path) -> RugpiTestResult<()> { - let case = toml::from_str::( - &fs::read_to_string(&case) +pub async fn main(project: &Project, workflow: &Path) -> RugpiTestResult<()> { + let workflow = toml::from_str::( + &fs::read_to_string(&workflow) .await - .whatever("unable to read test case")?, + .whatever("unable to read test workflow")?, ) - .whatever("unable to parse test case")?; - - let vm = qemu::start(&case.vm).await?; - - info!("VM started"); - - for step in &case.steps { - match step { - case::TestStep::Reboot => todo!(), - case::TestStep::Copy { .. } => todo!(), - case::TestStep::Run { - script, - stdin, - may_fail, - } => { - info!("running script"); - vm.wait_for_ssh() - .await - .whatever("unable to connect to VM via SSH")?; - if let Err(report) = vm - .run_script(script, stdin.as_ref().map(|p| p.as_ref())) - .await - .whatever::("unable to run script") - { - if may_fail.unwrap_or(false) { - eprintln!("ignoring error while executing script:\n{report:?}"); - } else { - bail!("error during test") + .whatever("unable to parse test workflow")?; + + for system in &workflow.systems { + let output = Path::new("build/images") + .join(&system.disk_image) + .with_extension("img"); + let project = project.clone(); + let disk_image = system.disk_image.clone(); + { + let output = output.clone(); + spawn_blocking(move || bake::bake_image(&project, &disk_image, &output)) + .await + .whatever("error baking image")? + .whatever("error baking image")?; + } + + let vm = qemu::start(&output.to_string_lossy(), system).await?; + + info!("VM started"); + + for step in &workflow.steps { + match step { + workflow::TestStep::Run { + script, + stdin, + may_fail, + } => { + info!("running script"); + vm.wait_for_ssh() + .await + .whatever("unable to connect to VM via SSH")?; + if let Err(report) = vm + .run_script(script, stdin.as_ref().map(|p| p.as_ref())) + .await + .whatever::("unable to run script") + { + if may_fail.unwrap_or(false) { + eprintln!("ignoring error while executing script:\n{report:?}"); + } else { + bail!("error during test") + } } } - } - TestStep::Wait { duration_secs } => { - info!("waiting for {duration_secs} seconds"); - tokio::time::sleep(Duration::from_secs_f64(*duration_secs)).await; + TestStep::Wait { duration } => { + info!("waiting for {duration} seconds"); + tokio::time::sleep(Duration::from_secs_f64(*duration)).await; + } } } } + Ok(()) } diff --git a/crates/rugpi-bakery/src/test/qemu.rs b/crates/rugpi-bakery/src/test/qemu.rs index 0dc71bd..2682281 100644 --- a/crates/rugpi-bakery/src/test/qemu.rs +++ b/crates/rugpi-bakery/src/test/qemu.rs @@ -18,7 +18,7 @@ use tokio::{ time, }; -use super::{case::VmConfig, RugpiTestResult}; +use super::{workflow::TestSystemConfig, RugpiTestResult}; pub struct Vm { #[expect(dead_code, reason = "not currently used")] @@ -26,7 +26,7 @@ pub struct Vm { ssh_session: Mutex>>, sftp_session: Mutex>, #[expect(dead_code, reason = "not currently used")] - vm_config: VmConfig, + vm_config: TestSystemConfig, private_key: Arc, } @@ -158,20 +158,20 @@ impl Vm { } } -pub async fn start(config: &VmConfig) -> RugpiTestResult { - let private_key = load_secret_key(&config.private_key, None) +pub async fn start(image_file: &str, config: &TestSystemConfig) -> RugpiTestResult { + let private_key = load_secret_key(&config.ssh.private_key, None) .whatever("unable to load private SSH key") - .with_info(|_| format!("path: {:?}", config.private_key))?; + .with_info(|_| format!("path: {:?}", config.ssh.private_key))?; fs::create_dir_all(".rugpi/") .await .whatever("unable to create .rugpi directory")?; if !Command::new("qemu-img") .args(&["create", "-f", "qcow2", "-F", "raw", "-o"]) - .arg(format!( - "backing_file=../{}", - config.image.to_string_lossy() - )) - .args(&[".rugpi/vm-image.img", "48G"]) + .arg(format!("backing_file=../{}", image_file)) + .args(&[ + ".rugpi/vm-image.img", + config.disk_size.as_deref().unwrap_or("40G"), + ]) .spawn() .whatever("unable to create VM image")? .wait() @@ -207,30 +207,22 @@ pub async fn start(config: &VmConfig) -> RugpiTestResult { ]); command .kill_on_drop(true) - .stdout(if config.stdout.is_some() { - Stdio::piped() - } else { - Stdio::null() - }) - .stderr(if config.stderr.is_some() { - Stdio::piped() - } else { - Stdio::null() - }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .stdin(Stdio::null()); let mut child = command.spawn().whatever("unable to spawn Qemu")?; - if let Some(stdout) = &config.stdout { + if let Some(stdout) = Some("build/vm-stdout.log") { let mut stdout_log = fs::File::create(stdout) .await .whatever("unable to create stdout log file")?; let mut stdout = child.stdout.take().expect("we used Stdio::piped"); tokio::spawn(async move { io::copy(&mut stdout, &mut stdout_log).await }); } - if let Some(stderr) = &config.stderr { + if let Some(stderr) = Some("build/vm-stderr.log") { let mut stderr_log = fs::File::create(stderr) .await .whatever("unable to create stderr log file")?; - let mut stderr = child.stdout.take().expect("we used Stdio::piped"); + let mut stderr = child.stderr.take().expect("we used Stdio::piped"); tokio::spawn(async move { io::copy(&mut stderr, &mut stderr_log).await }); } // We give Qemu some time to start before checking it's exit status. diff --git a/crates/rugpi-bakery/src/test/case.rs b/crates/rugpi-bakery/src/test/workflow.rs similarity index 65% rename from crates/rugpi-bakery/src/test/case.rs rename to crates/rugpi-bakery/src/test/workflow.rs index 4c6a361..9a09891 100644 --- a/crates/rugpi-bakery/src/test/case.rs +++ b/crates/rugpi-bakery/src/test/workflow.rs @@ -5,19 +5,28 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestCase { - pub vm: VmConfig, +pub struct TestWorkflow { + pub systems: Vec, pub steps: Vec, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct TestSystemConfig { + pub disk_image: String, + pub disk_size: Option, + pub ssh: SshConfig, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct SshConfig { + pub private_key: PathBuf, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "kebab-case", tag = "action")] pub enum TestStep { - Reboot, - Copy { - src: String, - dst: String, - }, #[serde(rename_all = "kebab-case")] Run { script: String, @@ -25,16 +34,5 @@ pub enum TestStep { may_fail: Option, }, #[serde(rename_all = "kebab-case")] - Wait { - duration_secs: f64, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct VmConfig { - pub image: PathBuf, - pub stdout: Option, - pub stderr: Option, - pub private_key: PathBuf, + Wait { duration: f64 }, } diff --git a/tests/tests/test-update.toml b/tests/tests/test-update.toml index ba3f151..7947c68 100644 --- a/tests/tests/test-update.toml +++ b/tests/tests/test-update.toml @@ -1,7 +1,6 @@ -[vm] -image = "./build/images/customized-arm64.img" -stdout = "./build/stdout.log" -private-key = "./files/id_rsa" +[[systems]] +disk-image = "customized-arm64" +ssh = { private-key = "./files/id_rsa" } [[steps]] action = "run" @@ -15,7 +14,7 @@ rugpi-ctrl update install - [[steps]] action = "wait" -duration-secs = 10.0 +duration = 10 [[steps]] action = "run" diff --git a/www/docs/advanced/integration-testing.md b/www/docs/advanced/integration-testing.md index b248d9d..090bb61 100644 --- a/www/docs/advanced/integration-testing.md +++ b/www/docs/advanced/integration-testing.md @@ -1,14 +1,70 @@ # Integration Testing -:::warning -**Work in progress!** See https://github.com/silitics/rugpi/issues/41. -::: +Embedded Linux systems are inherently complex, with numerous interconnected components working together. To ensure that all parts of a system work together seamlessly, Rugpi Bakery includes an integration testing framework designed to validate system images as a complete unit. This framework boots a system image in a virtual machine and then executes a _test workflow_ on the system. By catching integration errors early, it reduces the need for costly and time-consuming testing on physical hardware. -Embedded Linux systems are inherently complex, with numerous interconnected components working together. To ensure that all parts of a system work together seamlessly, Rugpi Bakery includes an integration testing framework designed to validate system images as a complete unit. This framework uses virtual machines to execute comprehensive _test workflows_. By catching integration errors early, it minimizes the need for costly and time-consuming testing on physical hardware. +**Limitations:** Currently, the integration testing framework is limited to the `generic-grub-efi` target. -Test workflows are placed in the `tests` directory of your Rugpi Bakery project. Each workflow consists of a TOML file describing the test to be conducted. -- TODO: We probably need some sort of testing matrix to test different configurations/images. +## Test Workflows + +Test workflows are placed in the `tests` directory of your Rugpi Bakery project. Each workflow consists of a TOML file describing the tests to be conducted. To this end, the workflow file starts with a declaration of _test systems_: + +```toml +[[systems]] +disk-image = "" +disk-size = "40G" +ssh = { private-key = "" } +``` + +Each test system declaration must specify a `disk-image`, which is the image to use for the system. In addition, a `disk-size` can be specified determining the size of the disk. Note that Rugpi Bakery allocates an image per system that grows on-demand and stores only the changes made over the original system image. Hence, typically much less than `disk-space` additional space is required. Multiple systems can be specified in the same test workflow. Rugpi will then run the workflow for each system. + +To execute commands on the system under test, Rugpi Bakery connects to the system running in the VM via SSH. To this end, a private key needs to specified. This private key must be placed in the project directory. It is recommended to generate a pair of keys exclusively for this purpose and inject the public key with an additional layer on-top of the actual system layer.[^1] To generate a suitable pair of SSH keys in the current working directory, run: + +```shell +ssh-keygen -t rsa -b 40960 -f id_rsa +``` + +The declaration of test systems is followed by a specification of _test steps_. + +[^1]: In the future, Rugpi Bakery may inject a key by itself prior to running the VM. + +### Test Steps + +Each test step performs a certain `action`. Currently, the following actions are supported: + +- `wait`: Wait for some amount of time. +- `run`: Run a script via SSH in the VM. + +#### Wait + +The `wait` action takes a `duration` option specifying the time to wait in seconds. Here is an example for waiting 20 seconds: + +```toml +[[steps]] +action = "wait" +duration = 20 +``` + +#### Run + +The `run` action takes a `script` option with a shell script to execute. For example: + +```toml +[[steps]] +action = "run" +script = """ +echo "Hello from the VM." +""" +``` + +In addition, the `run` action supports the following optional options: + +- `may-fail`: Indicates whether the script is allowed to fail. Normally, when a script fails, the corresponding test fails. Sometimes, e.g., when rebooting the system with a script, the execution may fail because the SSH connection drops, however, this is expected and the test should not fail. In this case, you can set `may-fail` to `true`. Note that a non-zero exit code of the script will always fail the test. +- `stdin`: Path to a file which is provided as stdin to the script. This is useful to stream, e.g., an update into the system. + +## Running Tests + +Tests can be run with the `test` subcommand: ```shell ./run-bakery test