From 4475ab61f61cf621280b4decc1bb78655b682947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20K=C3=B6hl?= Date: Wed, 25 Dec 2024 19:43:58 +0100 Subject: [PATCH] work on Rugpi's testing infrastructure (#41) --- crates/rugpi-test/src/case.rs | 1 + crates/rugpi-test/src/main.rs | 16 ++++-- crates/rugpi-test/src/qemu.rs | 24 +++++++-- tests/files/id_rsa | 49 +++++++++++++++++++ tests/files/id_rsa.pub | 1 + tests/layers/customized-efi.toml | 2 +- tests/recipes/setup-network/steps/01-packages | 3 +- tests/tests/test-update.toml | 14 +++--- www/docs/advanced/integration-testing.md | 15 ++++++ 9 files changed, 109 insertions(+), 16 deletions(-) create mode 100644 tests/files/id_rsa create mode 100644 tests/files/id_rsa.pub create mode 100644 www/docs/advanced/integration-testing.md diff --git a/crates/rugpi-test/src/case.rs b/crates/rugpi-test/src/case.rs index 93f611c..44af60b 100644 --- a/crates/rugpi-test/src/case.rs +++ b/crates/rugpi-test/src/case.rs @@ -21,6 +21,7 @@ pub enum TestStep { Run { script: String, stdin: Option, + may_fail: Option, }, Wait { duration_secs: f64, diff --git a/crates/rugpi-test/src/main.rs b/crates/rugpi-test/src/main.rs index b33585d..5435b11 100644 --- a/crates/rugpi-test/src/main.rs +++ b/crates/rugpi-test/src/main.rs @@ -2,7 +2,7 @@ use std::{path::PathBuf, time::Duration}; use case::{TestCase, TestStep}; use clap::Parser; -use reportify::{Report, ResultExt}; +use reportify::{bail, Report, ResultExt}; use rugpi_cli::info; use tokio::fs; @@ -36,13 +36,17 @@ pub async fn main() -> RugpiTestResult<()> { let vm = qemu::start(&case.vm).await?; - info!("VM running and connected via SSH"); + info!("VM started"); for step in &case.steps { match step { case::TestStep::Reboot => todo!(), case::TestStep::Copy { .. } => todo!(), - case::TestStep::Run { script, stdin } => { + case::TestStep::Run { + script, + stdin, + may_fail, + } => { info!("running script"); vm.wait_for_ssh() .await @@ -52,7 +56,11 @@ pub async fn main() -> RugpiTestResult<()> { .await .whatever::("unable to run script") { - eprintln!("{:?}", report); + if may_fail.unwrap_or(false) { + eprintln!("ignoring error while executing script:\n{report:?}"); + } else { + bail!("error during test") + } } } TestStep::Wait { duration_secs } => { diff --git a/crates/rugpi-test/src/qemu.rs b/crates/rugpi-test/src/qemu.rs index 296fa61..d90b719 100644 --- a/crates/rugpi-test/src/qemu.rs +++ b/crates/rugpi-test/src/qemu.rs @@ -162,6 +162,25 @@ pub async fn start(config: &VmConfig) -> RugpiTestResult { let private_key = load_secret_key(&config.private_key, None) .whatever("unable to load private SSH key") .with_info(|_| format!("path: {:?}", config.private_key))?; + fs::create_dir_all(".rugpi/") + .await + .whatever("unable to create .rugpi directory")?; + if !Command::new("qemu-img") + .args(&["create", "-f", "qcow2", "-F", "raw", "-o"]) + .arg(format!( + "backing_file=../{}", + config.image.to_string_lossy() + )) + .args(&[".rugpi/vm-image.img", "48G"]) + .spawn() + .whatever("unable to create VM image")? + .wait() + .await + .whatever("unable to create VM image")? + .success() + { + bail!("unable to create VM image"); + } let mut command = Command::new("qemu-system-aarch64"); command.args(&[ "-machine", @@ -174,10 +193,7 @@ pub async fn start(config: &VmConfig) -> RugpiTestResult { "cpus=2", ]); command.arg("-drive"); - command.arg(format!( - "file={},format=raw,if=virtio", - config.image.to_string_lossy() - )); + command.arg("file=.rugpi/vm-image.img,format=qcow2,if=virtio"); command.args(&["-device", "virtio-net-pci,netdev=net0", "-netdev"]); command.arg("user,id=net0,hostfwd=tcp:127.0.0.1:2233-:22"); command.args(&[ diff --git a/tests/files/id_rsa b/tests/files/id_rsa new file mode 100644 index 0000000..8bda1c4 --- /dev/null +++ b/tests/files/id_rsa @@ -0,0 +1,49 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEA1QsOSO42H9AHDG0pQlgbRFtPDkGdH/LkWsScy1ShR/a5LNW/DopT +FhcyGAjoMvGP9cRjek87BFOj10N/CVUN2v7zbVcfw2g0gaHTR4NgcJ5BAT91YV4Mnk8V2J +WRIPsDsZqhJpj1crPT97KdS7hBrC/du/F8yvA2fsPSx/C10iyMmKZjwNY/LPsz9dKPDPk4 +G0WhxTBhlPHzxNIYRnS4LHH5RucE0cyCEdKps7TKpXELbvbUG6ArJVOAW4J7MDrai53TgR +/GROX+b5yBLjuvr9tw7JxuIbSCgFmVYusZJ/KnfD2X51XtBf7Plq6ogTJxZJ2HxOWYJ/2L +vtHqQ+XuZ85beRl7vnpwZ0IQ/PFPUbNVjxmuwS4yhPsLvgmOxRWzw39bLmkC08iVypQ1gr +6ukLbcfGqD0hL3B4sgc69wzG/OpZjxL2GvQRWt0ZIyLar5A8NJAB9X8Tus/oXhGM+uHXTz +SJBDQ66Ycjj0yPK20xWO1XVkIJe0ybFIjP5o4XE+VZ9xXmwaIWiqI6OA9HHNk4Ou86vI+9 +QzORz0Mozz6hwPFfSWqLJF/VpCNcVZX6K1Vhu+kfLjvtr0rigGzuh9yPGGGPI1V8OCL9DR +tXry/i6uBeXT4ZjSaQaQ/yk6dk6KI+s2mH4g8enWpJ6Q2grgH9E/gz5V6C845VQBaqzuaF +8AAAdIeLICN3iyAjcAAAAHc3NoLXJzYQAAAgEA1QsOSO42H9AHDG0pQlgbRFtPDkGdH/Lk +WsScy1ShR/a5LNW/DopTFhcyGAjoMvGP9cRjek87BFOj10N/CVUN2v7zbVcfw2g0gaHTR4 +NgcJ5BAT91YV4Mnk8V2JWRIPsDsZqhJpj1crPT97KdS7hBrC/du/F8yvA2fsPSx/C10iyM +mKZjwNY/LPsz9dKPDPk4G0WhxTBhlPHzxNIYRnS4LHH5RucE0cyCEdKps7TKpXELbvbUG6 +ArJVOAW4J7MDrai53TgR/GROX+b5yBLjuvr9tw7JxuIbSCgFmVYusZJ/KnfD2X51XtBf7P +lq6ogTJxZJ2HxOWYJ/2LvtHqQ+XuZ85beRl7vnpwZ0IQ/PFPUbNVjxmuwS4yhPsLvgmOxR +Wzw39bLmkC08iVypQ1gr6ukLbcfGqD0hL3B4sgc69wzG/OpZjxL2GvQRWt0ZIyLar5A8NJ +AB9X8Tus/oXhGM+uHXTzSJBDQ66Ycjj0yPK20xWO1XVkIJe0ybFIjP5o4XE+VZ9xXmwaIW +iqI6OA9HHNk4Ou86vI+9QzORz0Mozz6hwPFfSWqLJF/VpCNcVZX6K1Vhu+kfLjvtr0rigG +zuh9yPGGGPI1V8OCL9DRtXry/i6uBeXT4ZjSaQaQ/yk6dk6KI+s2mH4g8enWpJ6Q2grgH9 +E/gz5V6C845VQBaqzuaF8AAAADAQABAAACAGGS7rf4YlGvtYq7dGoucE6eg1bENnKrjGsG +Icn+uN984NLoEtlWATfc59SaP78VZBnsGKOsJhuVMzUkdMqRagcjPdqWTLXyxBmbRGk8gq +8RzDQGHHy71ourLRZFanLwhICTkbtb0w71zd9kryY+/dUfFy89KyrHWi41AhUQ6jWYscw4 +aPepKMYxR1sPVPVcL5ZODgmCXy8Yoc1DbFrYmoHAsasHo+1y/R+snIzyAN5FKDdqVTtR4H +D4DZHb8Q/l5LKNVpGNPAHSVPZJd+D09UJV/yJfgxFs21htrz3e72acfJkqnKgXcreMPQeG +dya1M7ohbE2B2m52ccqz2de/N7D9lzVaoz92F82zQGBQzVIJtUIcpR4HHk9KrAvAHthbYK +JJ8YgO2sVgmDmTzhAU3yPOo9o0AXno8HWBmz/VFSJu0Taxgxnt6NzZL40OaBdRxM8g6gSt +JZokWSiErIJWzOcYpOYejLS+7BjP+UJ+xDoVKHKMHPs/Yeje2iM1rvktIzQra31KMUx27Z +QmVIkbnK7aQf3pAe2Hp/3Kg+mIJn4A3Kp605TZVKnvUB55TsQCaShZLUqQsh6hjjTDw/v4 +wOM5VLDexUgHScfDOlUyT+LsDU+hDKK8bjtlqfR8Ew1Q944p0fSm1EjXtzOGkKZMi0B5cj +gLnGR7i+S3eB1NHNQBAAABAEe8jWFwsWuoD6MnyIVBajrTxIgu4B/Fss6ZiyN4HmqpozTf +ayRkAhuKNGxEqcG50Ng9TOaT1OxBytaT6wGp6wPb24Im6U5FjqPZKrbT8IEws/qkN1qoG7 +qqIPwmjcJKwM/KFQgttmEp02nOrrZqjEYX35YNEHBlI5Y587/XUZfmFoqDWxF/mr3WtRo+ +Bs9C8SsdRAQy4U2Ypsy3yv4hHkxOr32dBFaK0fnt6/OKAaNtmb/ERWZI685BalE8dzuXe8 +1PVK9dCs0S3gz2xtl3wK8kZ/oJGm9ImsuH3QP9nTjoioy35S6kDHCINtKrPZHEYwEDtRs6 +CAACqpy14OnWF3MAAAEBAO8RWxEe/uBpnbuMB360UGlzes4QrbcbY7oN1u0PMBSu8RJkSX +pmJHUMT00Uu0pYYHpoxmPhhO1SPsEJt8QlKbvLqNImk/nCIRBqHMKHkzaePpfRuaLkTc12 +nyc3waLiiss6mr9cfQL7jBsGWkbwu9ZaQnw8I764+3fcQGuEn2dgOzAvJbPUC9GIlrJvXG +dHz6ls58XAM6u3OlxS0ZUOh8bGlPKKni1uzU0cZq7qvx7qj4FviYEEUkECNE6RC0jyGERa +pbHhyxFnClh43razLYgkmMnuA+MvKQTvwKQccwgsqyLpksHVWCyFLQ6hfcVLVCl6hptTjj +SbaIhrr/9+iZkAAAEBAOQh1hoqTvEpsFKLV42CJR4nTOkIA83pFoiJcnhh5daAv4D91jDo +YmbvVLGAipFozztLhCaaRJkAOeXIqpX3GjV7THI9IucVbPeDPmF/tLUR5IHEdt5XvQAPgg +ybFrzEfrFCyZ/jViCwWMb9AzVEyYVTE1gAFBOvlxZ2pwx1mSIwPcok8dksPIxD2BkdMqZP +oy3mr0jVR82w5TaEjRQkpjpPOvCYvBSWwB6qVL0S8spdydugDwMYVYIX8t+GsAAvwAJj25 +00Gw2n2GU5hIJuk40ERwzrSsz21WW/CuLwFPr/Uknv+CaoiHFyHyekvkZAFDZZCKnuSn3A +JW6LuhZJ7LcAAAAOcnVncGktdGVzdC1rZXkBAgMEBQ== +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/files/id_rsa.pub b/tests/files/id_rsa.pub new file mode 100644 index 0000000..234d878 --- /dev/null +++ b/tests/files/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVCw5I7jYf0AcMbSlCWBtEW08OQZ0f8uRaxJzLVKFH9rks1b8OilMWFzIYCOgy8Y/1xGN6TzsEU6PXQ38JVQ3a/vNtVx/DaDSBodNHg2BwnkEBP3VhXgyeTxXYlZEg+wOxmqEmmPVys9P3sp1LuEGsL9278XzK8DZ+w9LH8LXSLIyYpmPA1j8s+zP10o8M+TgbRaHFMGGU8fPE0hhGdLgscflG5wTRzIIR0qmztMqlcQtu9tQboCslU4BbgnswOtqLndOBH8ZE5f5vnIEuO6+v23DsnG4htIKAWZVi6xkn8qd8PZfnVe0F/s+WrqiBMnFknYfE5Zgn/Yu+0epD5e5nzlt5GXu+enBnQhD88U9Rs1WPGa7BLjKE+wu+CY7FFbPDf1suaQLTyJXKlDWCvq6Qttx8aoPSEvcHiyBzr3DMb86lmPEvYa9BFa3RkjItqvkDw0kAH1fxO6z+heEYz64ddPNIkENDrphyOPTI8rbTFY7VdWQgl7TJsUiM/mjhcT5Vn3FebBohaKojo4D0cc2Tg67zq8j71DM5HPQyjPPqHA8V9JaoskX9WkI1xVlforVWG76R8uO+2vSuKAbO6H3I8YYY8jVXw4Iv0NG1evL+Lq4F5dPhmNJpBpD/KTp2Tooj6zaYfiDx6daknpDaCuAf0T+DPlXoLzjlVAFqrO5oXw== rugpi-test-key diff --git a/tests/layers/customized-efi.toml b/tests/layers/customized-efi.toml index 71471ad..5c37c22 100644 --- a/tests/layers/customized-efi.toml +++ b/tests/layers/customized-efi.toml @@ -22,5 +22,5 @@ hostname = "rugpi-template" [parameters."core/ssh"] root_authorized_keys = """ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2tN+ZL8lGnx8rW+vm9ukX1uhKE7GREHitIVxIN3fVh406ksaZzY4FB7JqMqor4SBpR/Eigm6mSSE6KdwSYYC99hakLVvFUG6b6xvB7gOgre8M0JuL9XwBfaUfNln6Hl2Xirlml61JwOWa8Lh+T8mquw9OUK20tkXNPrigVKH+RMQA2V0AQrHnzo9GXMT5HEdAfaVVhL8S1inlKixiPbnvt+nWUSoKGLo+I08w5ZKI88C+saP6VqFiinp57uF0F3oqwcupZe0j6vxGuN5hFg8YGcICFnjXUAVjds8pfcf7aImvYZdp192jC9JAfzx3LzJZLn+kY9hIQkqip/tfTtp56eBb+j9i07PhrDsGiZVNOWf+YG3Cw5Ix6ltOL0dvF1/xFG7O+CGz62w8Y925ytuGgzBkVJ090eznnCjpw+lhNiNFmoqUjiJFjs/VSrqmC5bqdRrqF7YIs61uKl/EyAZaEoHZJazUFFauOjjPK0ksVbAAfqxG4nFmOG0URemSvNE= koehlma@Zaylee +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVCw5I7jYf0AcMbSlCWBtEW08OQZ0f8uRaxJzLVKFH9rks1b8OilMWFzIYCOgy8Y/1xGN6TzsEU6PXQ38JVQ3a/vNtVx/DaDSBodNHg2BwnkEBP3VhXgyeTxXYlZEg+wOxmqEmmPVys9P3sp1LuEGsL9278XzK8DZ+w9LH8LXSLIyYpmPA1j8s+zP10o8M+TgbRaHFMGGU8fPE0hhGdLgscflG5wTRzIIR0qmztMqlcQtu9tQboCslU4BbgnswOtqLndOBH8ZE5f5vnIEuO6+v23DsnG4htIKAWZVi6xkn8qd8PZfnVe0F/s+WrqiBMnFknYfE5Zgn/Yu+0epD5e5nzlt5GXu+enBnQhD88U9Rs1WPGa7BLjKE+wu+CY7FFbPDf1suaQLTyJXKlDWCvq6Qttx8aoPSEvcHiyBzr3DMb86lmPEvYa9BFa3RkjItqvkDw0kAH1fxO6z+heEYz64ddPNIkENDrphyOPTI8rbTFY7VdWQgl7TJsUiM/mjhcT5Vn3FebBohaKojo4D0cc2Tg67zq8j71DM5HPQyjPPqHA8V9JaoskX9WkI1xVlforVWG76R8uO+2vSuKAbO6H3I8YYY8jVXw4Iv0NG1evL+Lq4F5dPhmNJpBpD/KTp2Tooj6zaYfiDx6daknpDaCuAf0T+DPlXoLzjlVAFqrO5oXw== rugpi-test-key """ \ No newline at end of file diff --git a/tests/recipes/setup-network/steps/01-packages b/tests/recipes/setup-network/steps/01-packages index 11641a4..e1b2dc1 100644 --- a/tests/recipes/setup-network/steps/01-packages +++ b/tests/recipes/setup-network/steps/01-packages @@ -1 +1,2 @@ -pv \ No newline at end of file +pv +jq \ No newline at end of file diff --git a/tests/tests/test-update.toml b/tests/tests/test-update.toml index 9a984d1..a0fab08 100644 --- a/tests/tests/test-update.toml +++ b/tests/tests/test-update.toml @@ -1,14 +1,16 @@ [vm] -image = "./build/images/customized-arm64-vm.img" +image = "./build/images/customized-arm64.img" stdout = "./build/stdout.log" -private-key = "/Users/koehlma/.ssh/id_rsa" +private-key = "./files/id_rsa" [[steps]] action = "run" stdin = "./build/images/customized-arm64.img" +# Rebooting may cause the script to fail executing. We will check everything later. +may_fail = true script = """ -rugpi-ctrl system info -pv -f - | rugpi-ctrl update install - +rugpi-ctrl system info --json +rugpi-ctrl update install - """ [[steps]] @@ -18,7 +20,7 @@ duration_secs = 10.0 [[steps]] action = "run" script = """ -rugpi-ctrl system info +rugpi-ctrl system info --json rugpi-ctrl system commit -rugpi-ctrl system info +rugpi-ctrl system info --json """ \ No newline at end of file diff --git a/www/docs/advanced/integration-testing.md b/www/docs/advanced/integration-testing.md new file mode 100644 index 0000000..b248d9d --- /dev/null +++ b/www/docs/advanced/integration-testing.md @@ -0,0 +1,15 @@ +# Integration Testing + +:::warning +**Work in progress!** See https://github.com/silitics/rugpi/issues/41. +::: + +Embedded Linux systems are inherently complex, with numerous interconnected components working together. To ensure that all parts of a system work together seamlessly, Rugpi Bakery includes an integration testing framework designed to validate system images as a complete unit. This framework uses virtual machines to execute comprehensive _test workflows_. By catching integration errors early, it minimizes the need for costly and time-consuming testing on physical hardware. + +Test workflows are placed in the `tests` directory of your Rugpi Bakery project. Each workflow consists of a TOML file describing the test to be conducted. + +- TODO: We probably need some sort of testing matrix to test different configurations/images. + +```shell +./run-bakery test +``` \ No newline at end of file