work on Rugpi's testing infrastructure (#41)

crates/rugpi-test/src/case.rs

@@ -21,6 +21,7 @@ pub enum TestStep {
     Run {
         script: String,
         stdin: Option<PathBuf>,
+        may_fail: Option<bool>,
     },
     Wait {
         duration_secs: f64,

crates/rugpi-test/src/main.rs

@@ -2,7 +2,7 @@ use std::{path::PathBuf, time::Duration};
 
 use case::{TestCase, TestStep};
 use clap::Parser;
-use reportify::{Report, ResultExt};
+use reportify::{bail, Report, ResultExt};
 use rugpi_cli::info;
 use tokio::fs;
 
@@ -36,13 +36,17 @@ pub async fn main() -> RugpiTestResult<()> {
 
     let vm = qemu::start(&case.vm).await?;
 
-    info!("VM running and connected via SSH");
+    info!("VM started");
 
     for step in &case.steps {
         match step {
             case::TestStep::Reboot => todo!(),
             case::TestStep::Copy { .. } => todo!(),
-            case::TestStep::Run { script, stdin } => {
+            case::TestStep::Run {
+                script,
+                stdin,
+                may_fail,
+            } => {
                 info!("running script");
                 vm.wait_for_ssh()
                     .await
@@ -52,7 +56,11 @@ pub async fn main() -> RugpiTestResult<()> {
                     .await
                     .whatever::<RugpiTestError, _>("unable to run script")
                 {
-                    eprintln!("{:?}", report);
+                    if may_fail.unwrap_or(false) {
+                        eprintln!("ignoring error while executing script:\n{report:?}");
+                    } else {
+                        bail!("error during test")
+                    }
                 }
             }
             TestStep::Wait { duration_secs } => {

crates/rugpi-test/src/qemu.rs

@@ -162,6 +162,25 @@ pub async fn start(config: &VmConfig) -> RugpiTestResult<Vm> {
     let private_key = load_secret_key(&config.private_key, None)
         .whatever("unable to load private SSH key")
         .with_info(|_| format!("path: {:?}", config.private_key))?;
+    fs::create_dir_all(".rugpi/")
+        .await
+        .whatever("unable to create .rugpi directory")?;
+    if !Command::new("qemu-img")
+        .args(&["create", "-f", "qcow2", "-F", "raw", "-o"])
+        .arg(format!(
+            "backing_file=../{}",
+            config.image.to_string_lossy()
+        ))
+        .args(&[".rugpi/vm-image.img", "48G"])
+        .spawn()
+        .whatever("unable to create VM image")?
+        .wait()
+        .await
+        .whatever("unable to create VM image")?
+        .success()
+    {
+        bail!("unable to create VM image");
+    }
     let mut command = Command::new("qemu-system-aarch64");
     command.args(&[
         "-machine",
@@ -174,10 +193,7 @@ pub async fn start(config: &VmConfig) -> RugpiTestResult<Vm> {
         "cpus=2",
     ]);
     command.arg("-drive");
-    command.arg(format!(
-        "file={},format=raw,if=virtio",
-        config.image.to_string_lossy()
-    ));
+    command.arg("file=.rugpi/vm-image.img,format=qcow2,if=virtio");
     command.args(&["-device", "virtio-net-pci,netdev=net0", "-netdev"]);
     command.arg("user,id=net0,hostfwd=tcp:127.0.0.1:2233-:22");
     command.args(&[

tests/files/id_rsa

@@ -0,0 +1,49 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAgEA1QsOSO42H9AHDG0pQlgbRFtPDkGdH/LkWsScy1ShR/a5LNW/DopT
+FhcyGAjoMvGP9cRjek87BFOj10N/CVUN2v7zbVcfw2g0gaHTR4NgcJ5BAT91YV4Mnk8V2J
+WRIPsDsZqhJpj1crPT97KdS7hBrC/du/F8yvA2fsPSx/C10iyMmKZjwNY/LPsz9dKPDPk4
+G0WhxTBhlPHzxNIYRnS4LHH5RucE0cyCEdKps7TKpXELbvbUG6ArJVOAW4J7MDrai53TgR
+/GROX+b5yBLjuvr9tw7JxuIbSCgFmVYusZJ/KnfD2X51XtBf7Plq6ogTJxZJ2HxOWYJ/2L
+vtHqQ+XuZ85beRl7vnpwZ0IQ/PFPUbNVjxmuwS4yhPsLvgmOxRWzw39bLmkC08iVypQ1gr
+6ukLbcfGqD0hL3B4sgc69wzG/OpZjxL2GvQRWt0ZIyLar5A8NJAB9X8Tus/oXhGM+uHXTz
+SJBDQ66Ycjj0yPK20xWO1XVkIJe0ybFIjP5o4XE+VZ9xXmwaIWiqI6OA9HHNk4Ou86vI+9
+QzORz0Mozz6hwPFfSWqLJF/VpCNcVZX6K1Vhu+kfLjvtr0rigGzuh9yPGGGPI1V8OCL9DR
+tXry/i6uBeXT4ZjSaQaQ/yk6dk6KI+s2mH4g8enWpJ6Q2grgH9E/gz5V6C845VQBaqzuaF
+8AAAdIeLICN3iyAjcAAAAHc3NoLXJzYQAAAgEA1QsOSO42H9AHDG0pQlgbRFtPDkGdH/Lk
+WsScy1ShR/a5LNW/DopTFhcyGAjoMvGP9cRjek87BFOj10N/CVUN2v7zbVcfw2g0gaHTR4
+NgcJ5BAT91YV4Mnk8V2JWRIPsDsZqhJpj1crPT97KdS7hBrC/du/F8yvA2fsPSx/C10iyM
+mKZjwNY/LPsz9dKPDPk4G0WhxTBhlPHzxNIYRnS4LHH5RucE0cyCEdKps7TKpXELbvbUG6
+ArJVOAW4J7MDrai53TgR/GROX+b5yBLjuvr9tw7JxuIbSCgFmVYusZJ/KnfD2X51XtBf7P
+lq6ogTJxZJ2HxOWYJ/2LvtHqQ+XuZ85beRl7vnpwZ0IQ/PFPUbNVjxmuwS4yhPsLvgmOxR
+Wzw39bLmkC08iVypQ1gr6ukLbcfGqD0hL3B4sgc69wzG/OpZjxL2GvQRWt0ZIyLar5A8NJ
+AB9X8Tus/oXhGM+uHXTzSJBDQ66Ycjj0yPK20xWO1XVkIJe0ybFIjP5o4XE+VZ9xXmwaIW
+iqI6OA9HHNk4Ou86vI+9QzORz0Mozz6hwPFfSWqLJF/VpCNcVZX6K1Vhu+kfLjvtr0rigG
+zuh9yPGGGPI1V8OCL9DRtXry/i6uBeXT4ZjSaQaQ/yk6dk6KI+s2mH4g8enWpJ6Q2grgH9
+E/gz5V6C845VQBaqzuaF8AAAADAQABAAACAGGS7rf4YlGvtYq7dGoucE6eg1bENnKrjGsG
+Icn+uN984NLoEtlWATfc59SaP78VZBnsGKOsJhuVMzUkdMqRagcjPdqWTLXyxBmbRGk8gq
+8RzDQGHHy71ourLRZFanLwhICTkbtb0w71zd9kryY+/dUfFy89KyrHWi41AhUQ6jWYscw4
+aPepKMYxR1sPVPVcL5ZODgmCXy8Yoc1DbFrYmoHAsasHo+1y/R+snIzyAN5FKDdqVTtR4H
+D4DZHb8Q/l5LKNVpGNPAHSVPZJd+D09UJV/yJfgxFs21htrz3e72acfJkqnKgXcreMPQeG
+dya1M7ohbE2B2m52ccqz2de/N7D9lzVaoz92F82zQGBQzVIJtUIcpR4HHk9KrAvAHthbYK
+JJ8YgO2sVgmDmTzhAU3yPOo9o0AXno8HWBmz/VFSJu0Taxgxnt6NzZL40OaBdRxM8g6gSt
+JZokWSiErIJWzOcYpOYejLS+7BjP+UJ+xDoVKHKMHPs/Yeje2iM1rvktIzQra31KMUx27Z
+QmVIkbnK7aQf3pAe2Hp/3Kg+mIJn4A3Kp605TZVKnvUB55TsQCaShZLUqQsh6hjjTDw/v4
+wOM5VLDexUgHScfDOlUyT+LsDU+hDKK8bjtlqfR8Ew1Q944p0fSm1EjXtzOGkKZMi0B5cj
+gLnGR7i+S3eB1NHNQBAAABAEe8jWFwsWuoD6MnyIVBajrTxIgu4B/Fss6ZiyN4HmqpozTf
+ayRkAhuKNGxEqcG50Ng9TOaT1OxBytaT6wGp6wPb24Im6U5FjqPZKrbT8IEws/qkN1qoG7
+qqIPwmjcJKwM/KFQgttmEp02nOrrZqjEYX35YNEHBlI5Y587/XUZfmFoqDWxF/mr3WtRo+
+Bs9C8SsdRAQy4U2Ypsy3yv4hHkxOr32dBFaK0fnt6/OKAaNtmb/ERWZI685BalE8dzuXe8
+1PVK9dCs0S3gz2xtl3wK8kZ/oJGm9ImsuH3QP9nTjoioy35S6kDHCINtKrPZHEYwEDtRs6
+CAACqpy14OnWF3MAAAEBAO8RWxEe/uBpnbuMB360UGlzes4QrbcbY7oN1u0PMBSu8RJkSX
+pmJHUMT00Uu0pYYHpoxmPhhO1SPsEJt8QlKbvLqNImk/nCIRBqHMKHkzaePpfRuaLkTc12
+nyc3waLiiss6mr9cfQL7jBsGWkbwu9ZaQnw8I764+3fcQGuEn2dgOzAvJbPUC9GIlrJvXG
+dHz6ls58XAM6u3OlxS0ZUOh8bGlPKKni1uzU0cZq7qvx7qj4FviYEEUkECNE6RC0jyGERa
+pbHhyxFnClh43razLYgkmMnuA+MvKQTvwKQccwgsqyLpksHVWCyFLQ6hfcVLVCl6hptTjj
+SbaIhrr/9+iZkAAAEBAOQh1hoqTvEpsFKLV42CJR4nTOkIA83pFoiJcnhh5daAv4D91jDo
+YmbvVLGAipFozztLhCaaRJkAOeXIqpX3GjV7THI9IucVbPeDPmF/tLUR5IHEdt5XvQAPgg
+ybFrzEfrFCyZ/jViCwWMb9AzVEyYVTE1gAFBOvlxZ2pwx1mSIwPcok8dksPIxD2BkdMqZP
+oy3mr0jVR82w5TaEjRQkpjpPOvCYvBSWwB6qVL0S8spdydugDwMYVYIX8t+GsAAvwAJj25
+00Gw2n2GU5hIJuk40ERwzrSsz21WW/CuLwFPr/Uknv+CaoiHFyHyekvkZAFDZZCKnuSn3A
+JW6LuhZJ7LcAAAAOcnVncGktdGVzdC1rZXkBAgMEBQ==
+-----END OPENSSH PRIVATE KEY-----

tests/files/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVCw5I7jYf0AcMbSlCWBtEW08OQZ0f8uRaxJzLVKFH9rks1b8OilMWFzIYCOgy8Y/1xGN6TzsEU6PXQ38JVQ3a/vNtVx/DaDSBodNHg2BwnkEBP3VhXgyeTxXYlZEg+wOxmqEmmPVys9P3sp1LuEGsL9278XzK8DZ+w9LH8LXSLIyYpmPA1j8s+zP10o8M+TgbRaHFMGGU8fPE0hhGdLgscflG5wTRzIIR0qmztMqlcQtu9tQboCslU4BbgnswOtqLndOBH8ZE5f5vnIEuO6+v23DsnG4htIKAWZVi6xkn8qd8PZfnVe0F/s+WrqiBMnFknYfE5Zgn/Yu+0epD5e5nzlt5GXu+enBnQhD88U9Rs1WPGa7BLjKE+wu+CY7FFbPDf1suaQLTyJXKlDWCvq6Qttx8aoPSEvcHiyBzr3DMb86lmPEvYa9BFa3RkjItqvkDw0kAH1fxO6z+heEYz64ddPNIkENDrphyOPTI8rbTFY7VdWQgl7TJsUiM/mjhcT5Vn3FebBohaKojo4D0cc2Tg67zq8j71DM5HPQyjPPqHA8V9JaoskX9WkI1xVlforVWG76R8uO+2vSuKAbO6H3I8YYY8jVXw4Iv0NG1evL+Lq4F5dPhmNJpBpD/KTp2Tooj6zaYfiDx6daknpDaCuAf0T+DPlXoLzjlVAFqrO5oXw== rugpi-test-key

tests/layers/customized-efi.toml

@@ -22,5 +22,5 @@ hostname = "rugpi-template"
 
 [parameters."core/ssh"]
 root_authorized_keys = """
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2tN+ZL8lGnx8rW+vm9ukX1uhKE7GREHitIVxIN3fVh406ksaZzY4FB7JqMqor4SBpR/Eigm6mSSE6KdwSYYC99hakLVvFUG6b6xvB7gOgre8M0JuL9XwBfaUfNln6Hl2Xirlml61JwOWa8Lh+T8mquw9OUK20tkXNPrigVKH+RMQA2V0AQrHnzo9GXMT5HEdAfaVVhL8S1inlKixiPbnvt+nWUSoKGLo+I08w5ZKI88C+saP6VqFiinp57uF0F3oqwcupZe0j6vxGuN5hFg8YGcICFnjXUAVjds8pfcf7aImvYZdp192jC9JAfzx3LzJZLn+kY9hIQkqip/tfTtp56eBb+j9i07PhrDsGiZVNOWf+YG3Cw5Ix6ltOL0dvF1/xFG7O+CGz62w8Y925ytuGgzBkVJ090eznnCjpw+lhNiNFmoqUjiJFjs/VSrqmC5bqdRrqF7YIs61uKl/EyAZaEoHZJazUFFauOjjPK0ksVbAAfqxG4nFmOG0URemSvNE= koehlma@Zaylee
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVCw5I7jYf0AcMbSlCWBtEW08OQZ0f8uRaxJzLVKFH9rks1b8OilMWFzIYCOgy8Y/1xGN6TzsEU6PXQ38JVQ3a/vNtVx/DaDSBodNHg2BwnkEBP3VhXgyeTxXYlZEg+wOxmqEmmPVys9P3sp1LuEGsL9278XzK8DZ+w9LH8LXSLIyYpmPA1j8s+zP10o8M+TgbRaHFMGGU8fPE0hhGdLgscflG5wTRzIIR0qmztMqlcQtu9tQboCslU4BbgnswOtqLndOBH8ZE5f5vnIEuO6+v23DsnG4htIKAWZVi6xkn8qd8PZfnVe0F/s+WrqiBMnFknYfE5Zgn/Yu+0epD5e5nzlt5GXu+enBnQhD88U9Rs1WPGa7BLjKE+wu+CY7FFbPDf1suaQLTyJXKlDWCvq6Qttx8aoPSEvcHiyBzr3DMb86lmPEvYa9BFa3RkjItqvkDw0kAH1fxO6z+heEYz64ddPNIkENDrphyOPTI8rbTFY7VdWQgl7TJsUiM/mjhcT5Vn3FebBohaKojo4D0cc2Tg67zq8j71DM5HPQyjPPqHA8V9JaoskX9WkI1xVlforVWG76R8uO+2vSuKAbO6H3I8YYY8jVXw4Iv0NG1evL+Lq4F5dPhmNJpBpD/KTp2Tooj6zaYfiDx6daknpDaCuAf0T+DPlXoLzjlVAFqrO5oXw== rugpi-test-key
 """

tests/recipes/setup-network/steps/01-packages

@@ -1 +1,2 @@
-pv
+pv
+jq

tests/tests/test-update.toml

@@ -1,14 +1,16 @@
 [vm]
-image = "./build/images/customized-arm64-vm.img"
+image = "./build/images/customized-arm64.img"
 stdout = "./build/stdout.log"
-private-key = "/Users/koehlma/.ssh/id_rsa"
+private-key = "./files/id_rsa"
 
 [[steps]]
 action = "run"
 stdin = "./build/images/customized-arm64.img"
+# Rebooting may cause the script to fail executing. We will check everything later.
+may_fail = true
 script = """
-rugpi-ctrl system info
-pv -f - | rugpi-ctrl update install -
+rugpi-ctrl system info --json
+rugpi-ctrl update install -
 """
 
 [[steps]]
@@ -18,7 +20,7 @@ duration_secs = 10.0
 [[steps]]
 action = "run"
 script = """
-rugpi-ctrl system info
+rugpi-ctrl system info --json
 rugpi-ctrl system commit
-rugpi-ctrl system info
+rugpi-ctrl system info --json
 """

www/docs/advanced/integration-testing.md

@@ -0,0 +1,15 @@
+# Integration Testing
+
+:::warning
+**Work in progress!** See https://github.com/silitics/rugpi/issues/41.
+:::
+
+Embedded Linux systems are inherently complex, with numerous interconnected components working together. To ensure that all parts of a system work together seamlessly, Rugpi Bakery includes an integration testing framework designed to validate system images as a complete unit. This framework uses virtual machines to execute comprehensive _test workflows_. By catching integration errors early, it minimizes the need for costly and time-consuming testing on physical hardware.
+
+Test workflows are placed in the `tests` directory of your Rugpi Bakery project. Each workflow consists of a TOML file describing the test to be conducted.
+
+- TODO: We probably need some sort of testing matrix to test different configurations/images.
+
+```shell
+./run-bakery test
+```